[AArch64][llvm] Codegen for 16/32/64-bit floating-point atomicrmw fminimum/faximum (#137703)

Codegen for AArch64 16/32/64-bit floating-point atomic read-modify-write
operations (`atomicrmw {fmaximum,fminimum}`) using LD{B}FMAX and
LD{B}FMIN atomic instructions.

4 files changed, 244 insertions, 1191 deletions

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 63924dc..0d3cd57 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1001,6 +1001,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f64, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::bf16, LibCall);
+
+    setOperationAction(ISD::ATOMIC_LOAD_FMAXIMUM, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAXIMUM, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAXIMUM, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAXIMUM, MVT::bf16, LibCall);
+
+    setOperationAction(ISD::ATOMIC_LOAD_FMINIMUM, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMINIMUM, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMINIMUM, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMINIMUM, MVT::bf16, LibCall);
   }
 
   if (Subtarget->hasLSE128()) {
@@ -27991,7 +28001,9 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   // If LSFE available, use atomic FP instructions in preference to expansion
   if (Subtarget->hasLSFE() && (AI->getOperation() == AtomicRMWInst::FAdd ||
                                AI->getOperation() == AtomicRMWInst::FMax ||
-                               AI->getOperation() == AtomicRMWInst::FMin))
+                               AI->getOperation() == AtomicRMWInst::FMin ||
+                               AI->getOperation() == AtomicRMWInst::FMaximum ||
+                               AI->getOperation() == AtomicRMWInst::FMinimum))
     return AtomicExpansionKind::None;
 
   // Nand is not supported in LSE.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index f3734e0..31fcd63 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -551,14 +551,21 @@ defm atomic_load_fadd  : binary_atomic_op_fp<atomic_load_fadd>;
 defm atomic_load_fmin  : binary_atomic_op_fp<atomic_load_fmin>;
 defm atomic_load_fmax  : binary_atomic_op_fp<atomic_load_fmax>;
 
+defm atomic_load_fminimum  : binary_atomic_op_fp<atomic_load_fminimum>;
+defm atomic_load_fmaximum  : binary_atomic_op_fp<atomic_load_fmaximum>;
+
 let Predicates = [HasLSFE] in {
   defm : LDFPOPregister_patterns<"LDFADD",   "atomic_load_fadd">;
   defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">;
   defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">;
+  defm : LDFPOPregister_patterns<"LDFMAX",   "atomic_load_fmaximum">;
+  defm : LDFPOPregister_patterns<"LDFMIN",   "atomic_load_fminimum">;
 
   defm : LDBFPOPregister_patterns<"LDBFADD",   "atomic_load_fadd">;
   defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">;
   defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">;
+  defm : LDBFPOPregister_patterns<"LDBFMAX",   "atomic_load_fmaximum">;
+  defm : LDBFPOPregister_patterns<"LDBFMIN",   "atomic_load_fminimum">;
 }
 
 // v8.9a/v9.4a FEAT_LRCPC patterns
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index 7ee2a0b..d5ead2b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -1600,428 +1600,197 @@ define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, doubl
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
+; CHECK:    ldfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fmaximum_half_aligned_monotonic_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_monotonic_unused:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_monotonic_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_monotonic_unused:
+; CHECK:    ldfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value monotonic, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
+; CHECK:    ldfmaxa h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_release:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_release:
+; CHECK:    ldfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fmaximum_half_aligned_release_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_release_unused:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_release_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_release_unused:
+; CHECK:    ldfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value release, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
+; CHECK:    ldfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
+; CHECK:    ldfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
+; CHECK:    ldbfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fmaximum_bfloat_aligned_monotonic_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic_unused:
+; CHECK:    ldbfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
+; CHECK:    ldbfmaxa h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
+; CHECK:    ldbfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fmaximum_bfloat_aligned_release_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_release_unused:
+; CHECK:    ldbfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value release, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
+; CHECK:    ldbfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
+; CHECK:    ldbfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
+; CHECK:    ldfmax s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fmaximum_float_aligned_monotonic_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_monotonic_unused:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_monotonic_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_monotonic_unused:
+; CHECK:    ldfmax s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value monotonic, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
+; CHECK:    ldfmaxa s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_release:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_release:
+; CHECK:    ldfmaxl s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fmaximum_float_aligned_release_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_release_unused:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_release_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_release_unused:
+; CHECK:    ldfmaxl s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value release, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
+; CHECK:    ldfmaxal s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
+; CHECK:    ldfmaxal s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
+; CHECK:    ldfmax d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fmaximum_double_aligned_monotonic_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_monotonic_unused:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_monotonic_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_monotonic_unused:
+; CHECK:    ldfmax d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value monotonic, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
+; CHECK:    ldfmaxa d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_release:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_release:
+; CHECK:    ldfmaxl d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fmaximum_double_aligned_release_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_release_unused:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_release_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_release_unused:
+; CHECK:    ldfmaxl d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value release, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
+; CHECK:    ldfmaxal d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
+; CHECK:    ldfmaxal d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -2202,428 +1971,197 @@ define dso_local double @atomicrmw_fmaximum_double_unaligned_seq_cst(ptr %ptr, d
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
+; CHECK:    ldfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fminimum_half_aligned_monotonic_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_monotonic_unused:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_monotonic_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_monotonic_unused:
+; CHECK:    ldfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value monotonic, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_acquire:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_acquire:
+; CHECK:    ldfmina h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_release:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_release:
+; CHECK:    ldfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fminimum_half_aligned_release_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_release_unused:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_release_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_release_unused:
+; CHECK:    ldfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value release, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
+; CHECK:    ldfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
+; CHECK:    ldfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
+; CHECK:    ldbfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fminimum_bfloat_aligned_monotonic_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic_unused:
+; CHECK:    ldbfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
+; CHECK:    ldbfmina h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
+; CHECK:    ldbfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fminimum_bfloat_aligned_release_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_release_unused:
+; CHECK:    ldbfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value release, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
+; CHECK:    ldbfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
+; CHECK:    ldbfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
+; CHECK:    ldfmin s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fminimum_float_aligned_monotonic_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_monotonic_unused:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_monotonic_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_monotonic_unused:
+; CHECK:    ldfmin s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value monotonic, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_acquire:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_acquire:
+; CHECK:    ldfmina s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_release:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_release:
+; CHECK:    ldfminl s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fminimum_float_aligned_release_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_release_unused:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_release_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_release_unused:
+; CHECK:    ldfminl s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value release, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
+; CHECK:    ldfminal s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
+; CHECK:    ldfminal s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
+; CHECK:    ldfmin d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fminimum_double_aligned_monotonic_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_monotonic_unused:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_monotonic_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_monotonic_unused:
+; CHECK:    ldfmin d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value monotonic, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_acquire:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_acquire:
+; CHECK:    ldfmina d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_release:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_release:
+; CHECK:    ldfminl d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fminimum_double_aligned_release_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_release_unused:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_release_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_release_unused:
+; CHECK:    ldfminl d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value release, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
+; CHECK:    ldfminal d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
+; CHECK:    ldfminal d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
index 6940dab..4d1f3e9 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -1615,449 +1615,197 @@ define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, doubl
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_monotonic:
+; CHECK:    ldfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fmaximum_half_aligned_monotonic_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_monotonic_unused:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_monotonic_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_monotonic_unused:
+; CHECK:    ldfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value monotonic, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_acquire:
+; CHECK:    ldfmaxa h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_release:
+; CHECK:    ldfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fmaximum_half_aligned_release_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_release_unused:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_release_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_release_unused:
+; CHECK:    ldfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value release, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_acq_rel:
+; CHECK:    ldfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmaximum_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_half_aligned_seq_cst:
+; CHECK:    ldfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
+; CHECK:    ldbfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fmaximum_bfloat_aligned_monotonic_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic_unused:
+; CHECK:    ldbfmax h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
+; CHECK:    ldbfmaxa h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
+; CHECK:    ldbfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fmaximum_bfloat_aligned_release_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_release_unused:
+; CHECK:    ldbfmaxl h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value release, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
+; CHECK:    ldbfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
+; CHECK:    ldbfmaxal h0, h0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_monotonic:
+; CHECK:    ldfmax s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fmaximum_float_aligned_monotonic_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_monotonic_unused:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_monotonic_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_monotonic_unused:
+; CHECK:    ldfmax s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value monotonic, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_acquire:
+; CHECK:    ldfmaxa s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_release:
+; CHECK:    ldfmaxl s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fmaximum_float_aligned_release_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_release_unused:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_release_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_release_unused:
+; CHECK:    ldfmaxl s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value release, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_acq_rel:
+; CHECK:    ldfmaxal s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmaximum_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_float_aligned_seq_cst:
+; CHECK:    ldfmaxal s0, s0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_monotonic:
+; CHECK:    ldfmax d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fmaximum_double_aligned_monotonic_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_monotonic_unused:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_monotonic_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_monotonic_unused:
+; CHECK:    ldfmax d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value monotonic, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_acquire:
+; CHECK:    ldfmaxa d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_release:
+; CHECK:    ldfmaxl d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fmaximum_double_aligned_release_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_release_unused:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_release_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_release_unused:
+; CHECK:    ldfmaxl d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value release, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_acq_rel:
+; CHECK:    ldfmaxal d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmaximum_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmaximum_double_aligned_seq_cst:
+; CHECK:    ldfmaxal d0, d0, [x0]
     %r = atomicrmw fmaximum ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -2238,449 +1986,197 @@ define dso_local double @atomicrmw_fmaximum_double_unaligned_seq_cst(ptr %ptr, d
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_monotonic:
+; CHECK:    ldfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fminimum_half_aligned_monotonic_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_monotonic_unused:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_monotonic_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_monotonic_unused:
+; CHECK:    ldfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value monotonic, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_acquire:
+; CHECK:    ldfmina h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_release:
+; CHECK:    ldfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local void @atomicrmw_fminimum_half_aligned_release_unused(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_release_unused:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_release_unused:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_release_unused:
+; CHECK:    ldfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value release, align 2
     ret void
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_acq_rel:
+; CHECK:    ldfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fminimum_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_half_aligned_seq_cst:
+; CHECK:    ldfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
+; CHECK:    ldbfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fminimum_bfloat_aligned_monotonic_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic_unused:
+; CHECK:    ldbfmin h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
+; CHECK:    ldbfmina h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
+; CHECK:    ldbfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local void @atomicrmw_fminimum_bfloat_aligned_release_unused(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release_unused:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release_unused:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_release_unused:
+; CHECK:    ldbfminl h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value release, align 2
     ret void
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
+; CHECK:    ldbfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
+; CHECK:    ldbfminal h0, h0, [x0]
     %r = atomicrmw fminimum ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_monotonic:
+; CHECK:    ldfmin s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fminimum_float_aligned_monotonic_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_monotonic_unused:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_monotonic_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_monotonic_unused:
+; CHECK:    ldfmin s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value monotonic, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_acquire:
+; CHECK:    ldfmina s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_release:
+; CHECK:    ldfminl s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local void @atomicrmw_fminimum_float_aligned_release_unused(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_release_unused:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_release_unused:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_release_unused:
+; CHECK:    ldfminl s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value release, align 4
     ret void
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_acq_rel:
+; CHECK:    ldfminal s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fminimum_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_float_aligned_seq_cst:
+; CHECK:    ldfminal s0, s0, [x0]
     %r = atomicrmw fminimum ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_monotonic:
+; CHECK:    ldfmin d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fminimum_double_aligned_monotonic_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_monotonic_unused:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_monotonic_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_monotonic_unused:
+; CHECK:    ldfmin d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value monotonic, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_acquire:
+; CHECK:    ldfmina d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_release:
+; CHECK:    ldfminl d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local void @atomicrmw_fminimum_double_aligned_release_unused(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_release_unused:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_release_unused:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_release_unused:
+; CHECK:    ldfminl d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value release, align 8
     ret void
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_acq_rel:
+; CHECK:    ldfminal d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fminimum_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fminimum_double_aligned_seq_cst:
+; CHECK:    ldfminal d0, d0, [x0]
     %r = atomicrmw fminimum ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }