diff options
Diffstat (limited to 'llvm/test/CodeGen/AArch64')
21 files changed, 3978 insertions, 2361 deletions
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir index 09e5a15..a422f60 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir @@ -667,11 +667,10 @@ body: | ; SELECT-NEXT: {{ $}} ; SELECT-NEXT: %zero:gpr64 = COPY $xzr ; SELECT-NEXT: %reg0:gpr64 = COPY $x0 - ; SELECT-NEXT: %shl:gpr64 = UBFMXri %reg0, 1, 0 + ; SELECT-NEXT: %cmp_lhs:gpr64 = SUBSXrs %zero, %reg0, 63, implicit-def dead $nzcv ; SELECT-NEXT: %reg1:gpr64 = COPY $x1 ; SELECT-NEXT: %sext_in_reg:gpr64 = SBFMXri %reg1, 0, 0 - ; SELECT-NEXT: %cmp_rhs:gpr64 = SUBSXrs %zero, %sext_in_reg, 131, implicit-def dead $nzcv - ; SELECT-NEXT: [[ADDSXrr:%[0-9]+]]:gpr64 = ADDSXrr %shl, %cmp_rhs, implicit-def $nzcv + ; SELECT-NEXT: [[ADDSXrs:%[0-9]+]]:gpr64 = ADDSXrs %cmp_lhs, %sext_in_reg, 131, implicit-def $nzcv ; SELECT-NEXT: %cmp:gpr32 = CSINCWr $wzr, $wzr, 1, implicit $nzcv ; SELECT-NEXT: $w0 = COPY %cmp ; SELECT-NEXT: RET_ReallyLR implicit $w0 diff --git a/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll b/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll index 578038b..d9cdac4 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll @@ -1,8 +1,8 @@ ; RUN: llc -O3 -aarch64-enable-gep-opt=true -verify-machineinstrs %s -o - | FileCheck %s -; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare < %s 2>&1 | FileCheck --check-prefix=CHECK-UseAA %s -; RUN: llc -O3 -aarch64-enable-gep-opt=true -aarch64-use-aa=false -print-after=codegenprepare < %s 2>&1 | FileCheck --check-prefix=CHECK-NoAA %s -; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare -mcpu=cyclone < %s 2>&1 | FileCheck --check-prefix=CHECK-UseAA %s -; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare -mcpu=cortex-a53 < %s 2>&1 | FileCheck --check-prefix=CHECK-UseAA %s +; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare < %s 2>&1 | FileCheck --check-prefix=CHECK-IR %s +; RUN: llc -O3 -aarch64-enable-gep-opt=true -aarch64-use-aa=false -print-after=codegenprepare < %s 2>&1 | FileCheck --check-prefix=CHECK-IR %s +; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare -mcpu=cyclone < %s 2>&1 | FileCheck --check-prefix=CHECK-IR %s +; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare -mcpu=cortex-a53 < %s 2>&1 | FileCheck --check-prefix=CHECK-IR %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64" @@ -38,24 +38,12 @@ if.end: ; preds = %if.then, %entry ; CHECK-NOT: madd ; CHECK:ldr -; CHECK-NoAA-LABEL: @test_GEP_CSE( -; CHECK-NoAA: [[PTR0:%[a-zA-Z0-9]+]] = ptrtoint ptr %string to i64 -; CHECK-NoAA: [[PTR1:%[a-zA-Z0-9]+]] = mul i64 %idxprom, 96 -; CHECK-NoAA: [[PTR2:%[a-zA-Z0-9]+]] = add i64 [[PTR0]], [[PTR1]] -; CHECK-NoAA: add i64 [[PTR2]], 23052 -; CHECK-NoAA: inttoptr -; CHECK-NoAA: if.then: -; CHECK-NoAA-NOT: ptrtoint -; CHECK-NoAA-NOT: mul -; CHECK-NoAA: add i64 [[PTR2]], 23048 -; CHECK-NoAA: inttoptr - -; CHECK-UseAA-LABEL: @test_GEP_CSE( -; CHECK-UseAA: [[IDX:%[a-zA-Z0-9]+]] = mul i64 %idxprom, 96 -; CHECK-UseAA: [[PTR1:%[a-zA-Z0-9]+]] = getelementptr i8, ptr %string, i64 [[IDX]] -; CHECK-UseAA: getelementptr i8, ptr [[PTR1]], i64 23052 -; CHECK-UseAA: if.then: -; CHECK-UseAA: getelementptr i8, ptr [[PTR1]], i64 23048 +; CHECK-IR-LABEL: @test_GEP_CSE( +; CHECK-IR: [[IDX:%[a-zA-Z0-9]+]] = mul i64 %idxprom, 96 +; CHECK-IR: [[PTR1:%[a-zA-Z0-9]+]] = getelementptr i8, ptr %string, i64 [[IDX]] +; CHECK-IR: getelementptr i8, ptr [[PTR1]], i64 23052 +; CHECK-IR: if.then: +; CHECK-IR: getelementptr i8, ptr [[PTR1]], i64 23048 %class.my = type { i32, [128 x i32], i32, [256 x %struct.pt]} %struct.pt = type { ptr, i32, i32 } diff --git a/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll b/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll index f7e16b8..9947fba 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll @@ -38,14 +38,12 @@ define <16 x i32> @mul_i32(<16 x i8> %a, <16 x i8> %b) { ; ; CHECK-GI-LABEL: mul_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll2 v4.8h, v0.16b, #0 -; CHECK-GI-NEXT: ushll2 v5.8h, v1.16b, #0 -; CHECK-GI-NEXT: umull v0.4s, v2.4h, v3.4h -; CHECK-GI-NEXT: umull2 v1.4s, v2.8h, v3.8h -; CHECK-GI-NEXT: umull v2.4s, v4.4h, v5.4h -; CHECK-GI-NEXT: umull2 v3.4s, v4.8h, v5.8h +; CHECK-GI-NEXT: umull v2.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: umull2 v3.8h, v0.16b, v1.16b +; CHECK-GI-NEXT: ushll v0.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll v2.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0 ; CHECK-GI-NEXT: ret entry: %ea = zext <16 x i8> %a to <16 x i32> @@ -75,26 +73,20 @@ define <16 x i64> @mul_i64(<16 x i8> %a, <16 x i8> %b) { ; ; CHECK-GI-LABEL: mul_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-NEXT: ushll v4.4s, v2.4h, #0 -; CHECK-GI-NEXT: ushll2 v5.4s, v2.8h, #0 -; CHECK-GI-NEXT: ushll v2.4s, v3.4h, #0 -; CHECK-GI-NEXT: ushll v6.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0 -; CHECK-GI-NEXT: ushll v7.4s, v1.4h, #0 -; CHECK-GI-NEXT: ushll2 v16.4s, v0.8h, #0 -; CHECK-GI-NEXT: ushll2 v17.4s, v1.8h, #0 -; CHECK-GI-NEXT: umull v0.2d, v4.2s, v2.2s -; CHECK-GI-NEXT: umull2 v1.2d, v4.4s, v2.4s -; CHECK-GI-NEXT: umull v2.2d, v5.2s, v3.2s -; CHECK-GI-NEXT: umull2 v3.2d, v5.4s, v3.4s -; CHECK-GI-NEXT: umull v4.2d, v6.2s, v7.2s -; CHECK-GI-NEXT: umull2 v5.2d, v6.4s, v7.4s -; CHECK-GI-NEXT: umull v6.2d, v16.2s, v17.2s -; CHECK-GI-NEXT: umull2 v7.2d, v16.4s, v17.4s +; CHECK-GI-NEXT: umull v2.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: umull2 v0.8h, v0.16b, v1.16b +; CHECK-GI-NEXT: ushll v1.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll v5.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v7.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0 +; CHECK-GI-NEXT: ushll v2.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll2 v3.2d, v3.4s, #0 +; CHECK-GI-NEXT: ushll v4.2d, v5.2s, #0 +; CHECK-GI-NEXT: ushll2 v5.2d, v5.4s, #0 +; CHECK-GI-NEXT: ushll v6.2d, v7.2s, #0 +; CHECK-GI-NEXT: ushll2 v7.2d, v7.4s, #0 ; CHECK-GI-NEXT: ret entry: %ea = zext <16 x i8> %a to <16 x i64> @@ -142,18 +134,12 @@ define <16 x i32> @mla_i32(<16 x i8> %a, <16 x i8> %b, <16 x i32> %c) { ; ; CHECK-GI-LABEL: mla_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v6.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v7.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-NEXT: umlal v2.4s, v6.4h, v7.4h -; CHECK-GI-NEXT: umlal2 v3.4s, v6.8h, v7.8h -; CHECK-GI-NEXT: umlal v4.4s, v0.4h, v1.4h -; CHECK-GI-NEXT: umlal2 v5.4s, v0.8h, v1.8h -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: mov v1.16b, v3.16b -; CHECK-GI-NEXT: mov v2.16b, v4.16b -; CHECK-GI-NEXT: mov v3.16b, v5.16b +; CHECK-GI-NEXT: umull v6.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: umull2 v7.8h, v0.16b, v1.16b +; CHECK-GI-NEXT: uaddw v0.4s, v2.4s, v6.4h +; CHECK-GI-NEXT: uaddw2 v1.4s, v3.4s, v6.8h +; CHECK-GI-NEXT: uaddw v2.4s, v4.4s, v7.4h +; CHECK-GI-NEXT: uaddw2 v3.4s, v5.4s, v7.8h ; CHECK-GI-NEXT: ret entry: %ea = zext <16 x i8> %a to <16 x i32> @@ -186,35 +172,21 @@ define <16 x i64> @mla_i64(<16 x i8> %a, <16 x i8> %b, <16 x i64> %c) { ; ; CHECK-GI-LABEL: mla_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mov v16.16b, v2.16b -; CHECK-GI-NEXT: mov v17.16b, v3.16b -; CHECK-GI-NEXT: mov v2.16b, v4.16b -; CHECK-GI-NEXT: mov v3.16b, v5.16b -; CHECK-GI-NEXT: mov v4.16b, v6.16b -; CHECK-GI-NEXT: mov v5.16b, v7.16b -; CHECK-GI-NEXT: ushll v6.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v7.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-NEXT: ushll v18.4s, v6.4h, #0 -; CHECK-GI-NEXT: ushll v20.4s, v7.4h, #0 -; CHECK-GI-NEXT: ushll2 v19.4s, v6.8h, #0 -; CHECK-GI-NEXT: ushll v21.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll2 v22.4s, v7.8h, #0 -; CHECK-GI-NEXT: ushll v23.4s, v1.4h, #0 -; CHECK-GI-NEXT: ldp q6, q7, [sp] -; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: umlal v16.2d, v18.2s, v20.2s -; CHECK-GI-NEXT: umlal2 v17.2d, v18.4s, v20.4s -; CHECK-GI-NEXT: umlal v2.2d, v19.2s, v22.2s -; CHECK-GI-NEXT: umlal2 v3.2d, v19.4s, v22.4s -; CHECK-GI-NEXT: umlal v4.2d, v21.2s, v23.2s -; CHECK-GI-NEXT: umlal2 v5.2d, v21.4s, v23.4s -; CHECK-GI-NEXT: umlal v6.2d, v0.2s, v1.2s -; CHECK-GI-NEXT: umlal2 v7.2d, v0.4s, v1.4s -; CHECK-GI-NEXT: mov v0.16b, v16.16b -; CHECK-GI-NEXT: mov v1.16b, v17.16b +; CHECK-GI-NEXT: umull v16.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: umull2 v0.8h, v0.16b, v1.16b +; CHECK-GI-NEXT: ldp q19, q20, [sp] +; CHECK-GI-NEXT: ushll v1.4s, v16.4h, #0 +; CHECK-GI-NEXT: ushll2 v16.4s, v16.8h, #0 +; CHECK-GI-NEXT: ushll v17.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v18.4s, v0.8h, #0 +; CHECK-GI-NEXT: uaddw v0.2d, v2.2d, v1.2s +; CHECK-GI-NEXT: uaddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: uaddw v2.2d, v4.2d, v16.2s +; CHECK-GI-NEXT: uaddw2 v3.2d, v5.2d, v16.4s +; CHECK-GI-NEXT: uaddw v4.2d, v6.2d, v17.2s +; CHECK-GI-NEXT: uaddw2 v5.2d, v7.2d, v17.4s +; CHECK-GI-NEXT: uaddw v6.2d, v19.2d, v18.2s +; CHECK-GI-NEXT: uaddw2 v7.2d, v20.2d, v18.4s ; CHECK-GI-NEXT: ret entry: %ea = zext <16 x i8> %a to <16 x i64> diff --git a/llvm/test/CodeGen/AArch64/adc.ll b/llvm/test/CodeGen/AArch64/adc.ll index 4b1393f..12e8bf2 100644 --- a/llvm/test/CodeGen/AArch64/adc.ll +++ b/llvm/test/CodeGen/AArch64/adc.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -verify-machineinstrs < %s -mtriple=arm64-apple-ios7.0 | FileCheck --check-prefix=CHECK-LE %s -; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64_be-none-linux-gnu | FileCheck --check-prefix=CHECK-BE %s +; RUN: llc -verify-machineinstrs < %s -mtriple=arm64-apple-ios7.0 | FileCheck --check-prefixes=CHECK-LE %s +; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64_be-none-linux-gnu | FileCheck --check-prefixes=CHECK-BE %s +; RUN: llc -verify-machineinstrs < %s -mtriple=arm64-apple-ios7.0 -global-isel | FileCheck --check-prefixes=CHECK-GI %s define i128 @test_simple(i128 %a, i128 %b, i128 %c) { ; CHECK-LE-LABEL: test_simple: @@ -18,11 +19,16 @@ define i128 @test_simple(i128 %a, i128 %b, i128 %c) { ; CHECK-BE-NEXT: subs x1, x8, x5 ; CHECK-BE-NEXT: sbc x0, x9, x4 ; CHECK-BE-NEXT: ret - +; +; CHECK-GI-LABEL: test_simple: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: adds x8, x0, x2 +; CHECK-GI-NEXT: adc x9, x1, x3 +; CHECK-GI-NEXT: subs x0, x8, x4 +; CHECK-GI-NEXT: sbc x1, x9, x5 +; CHECK-GI-NEXT: ret %valadd = add i128 %a, %b - %valsub = sub i128 %valadd, %c - ret i128 %valsub } @@ -38,9 +44,13 @@ define i128 @test_imm(i128 %a) { ; CHECK-BE-NEXT: adds x1, x1, #12 ; CHECK-BE-NEXT: cinc x0, x0, hs ; CHECK-BE-NEXT: ret - +; +; CHECK-GI-LABEL: test_imm: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: adds x0, x0, #12 +; CHECK-GI-NEXT: adc x1, x1, xzr +; CHECK-GI-NEXT: ret %val = add i128 %a, 12 - ret i128 %val } @@ -58,11 +68,16 @@ define i128 @test_shifted(i128 %a, i128 %b) { ; CHECK-BE-NEXT: adds x1, x1, x3, lsl #45 ; CHECK-BE-NEXT: adc x0, x0, x8 ; CHECK-BE-NEXT: ret - +; +; CHECK-GI-LABEL: test_shifted: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: lsr x8, x2, #19 +; CHECK-GI-NEXT: adds x0, x0, x2, lsl #45 +; CHECK-GI-NEXT: orr x8, x8, x3, lsl #45 +; CHECK-GI-NEXT: adc x1, x1, x8 +; CHECK-GI-NEXT: ret %rhs = shl i128 %b, 45 - %val = add i128 %a, %rhs - ret i128 %val } @@ -86,11 +101,19 @@ define i128 @test_extended(i128 %a, i16 %b) { ; CHECK-BE-NEXT: extr x8, x9, x8, #61 ; CHECK-BE-NEXT: adc x0, x0, x8 ; CHECK-BE-NEXT: ret - +; +; CHECK-GI-LABEL: test_extended: +; CHECK-GI: ; %bb.0: +; CHECK-GI-NEXT: ; kill: def $w2 killed $w2 def $x2 +; CHECK-GI-NEXT: sxth x8, w2 +; CHECK-GI-NEXT: adds x0, x0, w2, sxth #3 +; CHECK-GI-NEXT: asr x9, x8, #63 +; CHECK-GI-NEXT: lsr x8, x8, #61 +; CHECK-GI-NEXT: orr x8, x8, x9, lsl #3 +; CHECK-GI-NEXT: adc x1, x1, x8 +; CHECK-GI-NEXT: ret %ext = sext i16 %b to i128 %rhs = shl i128 %ext, 3 - %val = add i128 %a, %rhs - ret i128 %val } diff --git a/llvm/test/CodeGen/AArch64/addcarry-crash.ll b/llvm/test/CodeGen/AArch64/addcarry-crash.ll index be75ab1..b4556c7 100644 --- a/llvm/test/CodeGen/AArch64/addcarry-crash.ll +++ b/llvm/test/CodeGen/AArch64/addcarry-crash.ll @@ -1,16 +1,29 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s | FileCheck %s +; RUN: llc < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI + target triple = "arm64-apple-ios7.0" define i64 @foo(ptr nocapture readonly %ptr, i64 %a, i64 %b, i64 %c) local_unnamed_addr #0 { -; CHECK-LABEL: foo: -; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: lsr x8, x1, #32 -; CHECK-NEXT: ldr w9, [x0, #4] -; CHECK-NEXT: cmn x3, x2 -; CHECK-NEXT: umull x8, w9, w8 -; CHECK-NEXT: cinc x0, x8, hs -; CHECK-NEXT: ret +; CHECK-SD-LABEL: foo: +; CHECK-SD: ; %bb.0: ; %entry +; CHECK-SD-NEXT: lsr x8, x1, #32 +; CHECK-SD-NEXT: ldr w9, [x0, #4] +; CHECK-SD-NEXT: cmn x3, x2 +; CHECK-SD-NEXT: umull x8, w9, w8 +; CHECK-SD-NEXT: cinc x0, x8, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: foo: +; CHECK-GI: ; %bb.0: ; %entry +; CHECK-GI-NEXT: ldr x8, [x0] +; CHECK-GI-NEXT: lsr x9, x1, #32 +; CHECK-GI-NEXT: cmn x3, x2 +; CHECK-GI-NEXT: cset w10, hs +; CHECK-GI-NEXT: lsr x8, x8, #32 +; CHECK-GI-NEXT: and x10, x10, #0x1 +; CHECK-GI-NEXT: umaddl x0, w8, w9, x10 +; CHECK-GI-NEXT: ret entry: %0 = lshr i64 %a, 32 %1 = load i64, ptr %ptr, align 8 @@ -24,3 +37,6 @@ entry: } attributes #0 = { norecurse nounwind readonly } + +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}} diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll index b325851..78881c8 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck -check-prefixes=CHECK,CHECK-SD %s +; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s -check-prefixes=CHECK,CHECK-SD ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI define <8 x i16> @sabdl8h(ptr %A, ptr %B) nounwind { diff --git a/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll b/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll index 634d1b9..5f5b27a 100644 --- a/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll +++ b/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll @@ -59,37 +59,33 @@ bb27: ; preds = %bb9, %bb8 define void @avoid_promotion_2_and(ptr nocapture noundef %arg) { ; CHECK-LABEL: avoid_promotion_2_and: ; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: add x8, x0, #32 -; CHECK-NEXT: b LBB1_2 -; CHECK-NEXT: LBB1_1: ; %latch -; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1 -; CHECK-NEXT: cmp w9, #2 -; CHECK-NEXT: add x8, x8, #56 -; CHECK-NEXT: b.ls LBB1_4 -; CHECK-NEXT: LBB1_2: ; %loop +; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: add x9, x0, #32 +; CHECK-NEXT: LBB1_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr w9, [x8, #20] -; CHECK-NEXT: cmp w9, #3 -; CHECK-NEXT: b.lo LBB1_1 -; CHECK-NEXT: ; %bb.3: ; %then -; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1 -; CHECK-NEXT: ldp w13, w12, [x8, #12] -; CHECK-NEXT: ldr w10, [x8] +; CHECK-NEXT: ldr w10, [x9, #20] +; CHECK-NEXT: cmp w10, #3 +; CHECK-NEXT: b.lo LBB1_3 +; CHECK-NEXT: ; %bb.2: ; %then +; CHECK-NEXT: ; in Loop: Header=BB1_1 Depth=1 +; CHECK-NEXT: ldp w13, w12, [x9, #12] +; CHECK-NEXT: ldr w10, [x9] ; CHECK-NEXT: ldr x11, [x0] -; CHECK-NEXT: ldr w14, [x8, #8] +; CHECK-NEXT: add x8, x8, #1 +; CHECK-NEXT: ldr w14, [x9, #8] ; CHECK-NEXT: lsl w10, w10, w13 ; CHECK-NEXT: ldrb w11, [x11, x12] ; CHECK-NEXT: eor w10, w10, w11 -; CHECK-NEXT: ldur w11, [x8, #-24] +; CHECK-NEXT: ldur w11, [x9, #-24] ; CHECK-NEXT: and w10, w10, w14 -; CHECK-NEXT: ldp x14, x13, [x8, #-16] -; CHECK-NEXT: str w10, [x8] +; CHECK-NEXT: ldp x14, x13, [x9, #-16] +; CHECK-NEXT: str w10, [x9], #56 ; CHECK-NEXT: and w11, w11, w12 ; CHECK-NEXT: ldrh w15, [x13, w10, uxtw #1] ; CHECK-NEXT: strh w15, [x14, w11, uxtw #1] ; CHECK-NEXT: strh w12, [x13, w10, uxtw #1] ; CHECK-NEXT: b LBB1_1 -; CHECK-NEXT: LBB1_4: ; %exit +; CHECK-NEXT: LBB1_3: ; %exit.critedge ; CHECK-NEXT: ret entry: br label %loop diff --git a/llvm/test/CodeGen/AArch64/calleetypeid-directcall-mismatched.ll b/llvm/test/CodeGen/AArch64/calleetypeid-directcall-mismatched.ll new file mode 100644 index 0000000..c4c54175 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/calleetypeid-directcall-mismatched.ll @@ -0,0 +1,32 @@ +;; Tests that callee_type metadata attached to direct call sites are safely ignored. + +; RUN: llc --call-graph-section -mtriple aarch64-linux-gnu < %s -stop-after=finalize-isel -o - | FileCheck --match-full-lines %s + +;; Test that `calleeTypeIds` field is not present in `callSites` +; CHECK-LABEL: callSites: +; CHECK-NEXT: - { bb: {{[0-9]+}}, offset: {{[0-9]+}}, fwdArgRegs: [] } +; CHECK-NEXT: - { bb: {{[0-9]+}}, offset: {{[0-9]+}}, fwdArgRegs: [] } +; CHECK-NEXT: - { bb: {{[0-9]+}}, offset: {{[0-9]+}}, fwdArgRegs: [] } +define i32 @foo(i32 %x, i32 %y) !type !0 { +entry: + ;; Call instruction with accurate callee_type. + ;; callee_type should be dropped seemlessly. + %call = call i32 @fizz(i32 %x, i32 %y), !callee_type !1 + ;; Call instruction with mismatched callee_type. + ;; callee_type should be dropped seemlessly without errors. + %call1 = call i32 @fizz(i32 %x, i32 %y), !callee_type !3 + %add = add nsw i32 %call, %call1 + ;; Call instruction with mismatched callee_type. + ;; callee_type should be dropped seemlessly without errors. + %call2 = call i32 @fizz(i32 %add, i32 %y), !callee_type !3 + %sub = sub nsw i32 %add, %call2 + ret i32 %sub +} + +declare !type !2 i32 @fizz(i32, i32) + +!0 = !{i64 0, !"_ZTSFiiiiE.generalized"} +!1 = !{!2} +!2 = !{i64 0, !"_ZTSFiiiE.generalized"} +!3 = !{!4} +!4 = !{i64 0, !"_ZTSFicE.generalized"} diff --git a/llvm/test/CodeGen/AArch64/callsite-emit-calleetypeid-tailcall.ll b/llvm/test/CodeGen/AArch64/callsite-emit-calleetypeid-tailcall.ll new file mode 100644 index 0000000..b47607e --- /dev/null +++ b/llvm/test/CodeGen/AArch64/callsite-emit-calleetypeid-tailcall.ll @@ -0,0 +1,19 @@ +;; Tests that call site callee type ids can be extracted and set from +;; callee_type metadata for indirect tail calls. + +;; Verify the exact calleeTypeId value to ensure it is not garbage but the value +;; computed as the type id from the callee_type metadata. +; RUN: llc --call-graph-section -mtriple aarch64-linux-gnu < %s -stop-after=finalize-isel -o - | FileCheck --match-full-lines %s + +define i32 @check_tailcall(ptr %func, i8 %x) !type !0 { +entry: + ; CHECK: callSites: + ; CHECK-NEXT: - { bb: {{.*}}, offset: {{.*}}, fwdArgRegs: [], calleeTypeIds: + ; CHECK-NEXT: [ 3498816979441845844 ] } + %call = tail call i32 %func(i8 signext %x), !callee_type !1 + ret i32 %call +} + +!0 = !{i64 0, !"_ZTSFiPvcE.generalized"} +!1 = !{!2} +!2 = !{i64 0, !"_ZTSFicE.generalized"} diff --git a/llvm/test/CodeGen/AArch64/callsite-emit-calleetypeid.ll b/llvm/test/CodeGen/AArch64/callsite-emit-calleetypeid.ll new file mode 100644 index 0000000..94b657c --- /dev/null +++ b/llvm/test/CodeGen/AArch64/callsite-emit-calleetypeid.ll @@ -0,0 +1,20 @@ +;; Tests that call site callee type ids can be extracted and set from +;; callee_type metadata. + +;; Verify the exact calleeTypeIds value to ensure it is not garbage but the value +;; computed as the type id from the callee_type metadata. +; RUN: llc --call-graph-section -mtriple aarch64-linux-gnu < %s -stop-after=finalize-isel -o - | FileCheck --match-full-lines %s + +; CHECK: name: main +; CHECK: callSites: +; CHECK-NEXT: - { bb: {{.*}}, offset: {{.*}}, fwdArgRegs: [], calleeTypeIds: +; CHECK-NEXT: [ 7854600665770582568 ] } +define i32 @main() { +entry: + %fn = load ptr, ptr null, align 8 + call void %fn(i8 0), !callee_type !0 + ret i32 0 +} + +!0 = !{!1} +!1 = !{i64 0, !"_ZTSFvcE.generalized"} diff --git a/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll b/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll index 5765e0a..b3ce9d2 100644 --- a/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll +++ b/llvm/test/CodeGen/AArch64/cmp-to-cmn.ll @@ -1,14 +1,21 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "arm64" define i1 @test_EQ_IllEbT(i64 %a, i64 %b) { -; CHECK-LABEL: test_EQ_IllEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn x0, x1 -; CHECK-NEXT: cset w0, eq -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_EQ_IllEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn x0, x1 +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_EQ_IllEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmn x1, x0 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret entry: %add = sub i64 0, %b %cmp = icmp eq i64 %add, %a @@ -16,11 +23,19 @@ entry: } define i1 @test_EQ_IliEbT(i64 %a, i32 %b) { -; CHECK-LABEL: test_EQ_IliEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn x0, w1, sxtw -; CHECK-NEXT: cset w0, eq -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_EQ_IliEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn x0, w1, sxtw +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_EQ_IliEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-GI-NEXT: sxtw x8, w1 +; CHECK-GI-NEXT: cmn x8, x0 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret entry: %conv = sext i32 %b to i64 %add = sub i64 0, %a @@ -55,11 +70,19 @@ entry: } define i1 @test_EQ_IilEbT(i32 %a, i64 %b) { -; CHECK-LABEL: test_EQ_IilEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn x1, w0, sxtw -; CHECK-NEXT: cset w0, eq -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_EQ_IilEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn x1, w0, sxtw +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_EQ_IilEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-GI-NEXT: sxtw x8, w0 +; CHECK-GI-NEXT: cmn x8, x1 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret entry: %conv = sext i32 %a to i64 %add = sub i64 0, %b @@ -68,11 +91,17 @@ entry: } define i1 @test_EQ_IiiEbT(i32 %a, i32 %b) { -; CHECK-LABEL: test_EQ_IiiEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn w0, w1 -; CHECK-NEXT: cset w0, eq -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_EQ_IiiEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn w0, w1 +; CHECK-SD-NEXT: cset w0, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_EQ_IiiEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmn w1, w0 +; CHECK-GI-NEXT: cset w0, eq +; CHECK-GI-NEXT: ret entry: %add = sub i32 0, %b %cmp = icmp eq i32 %add, %a @@ -218,11 +247,17 @@ entry: } define i1 @test_NE_IllEbT(i64 %a, i64 %b) { -; CHECK-LABEL: test_NE_IllEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn x0, x1 -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_NE_IllEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn x0, x1 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_NE_IllEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmn x1, x0 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret entry: %add = sub i64 0, %b %cmp = icmp ne i64 %add, %a @@ -230,11 +265,19 @@ entry: } define i1 @test_NE_IliEbT(i64 %a, i32 %b) { -; CHECK-LABEL: test_NE_IliEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn x0, w1, sxtw -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_NE_IliEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn x0, w1, sxtw +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_NE_IliEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-GI-NEXT: sxtw x8, w1 +; CHECK-GI-NEXT: cmn x8, x0 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret entry: %conv = sext i32 %b to i64 %add = sub i64 0, %a @@ -269,11 +312,19 @@ entry: } define i1 @test_NE_IilEbT(i32 %a, i64 %b) { -; CHECK-LABEL: test_NE_IilEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn x1, w0, sxtw -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_NE_IilEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn x1, w0, sxtw +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_NE_IilEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-GI-NEXT: sxtw x8, w0 +; CHECK-GI-NEXT: cmn x8, x1 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret entry: %conv = sext i32 %a to i64 %add = sub i64 0, %b @@ -282,11 +333,17 @@ entry: } define i1 @test_NE_IiiEbT(i32 %a, i32 %b) { -; CHECK-LABEL: test_NE_IiiEbT: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cmn w0, w1 -; CHECK-NEXT: cset w0, ne -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_NE_IiiEbT: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: cmn w0, w1 +; CHECK-SD-NEXT: cset w0, ne +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_NE_IiiEbT: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: cmn w1, w0 +; CHECK-GI-NEXT: cset w0, ne +; CHECK-GI-NEXT: ret entry: %add = sub i32 0, %b %cmp = icmp ne i32 %add, %a @@ -444,161 +501,281 @@ define i1 @cmn_large_imm(i32 %a) { } define i1 @almost_immediate_neg_slt(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_slt: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, le -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_slt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_slt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #4097 // =0x1001 +; CHECK-GI-NEXT: movk w8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, lt +; CHECK-GI-NEXT: ret %cmp = icmp slt i32 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_slt_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_slt_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, le -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_slt_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, le +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_slt_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-61439 // =0xffffffffffff1001 +; CHECK-GI-NEXT: movk x8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, lt +; CHECK-GI-NEXT: ret %cmp = icmp slt i64 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_sge(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_sge: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, gt -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_sge: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_sge: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #4097 // =0x1001 +; CHECK-GI-NEXT: movk w8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, ge +; CHECK-GI-NEXT: ret %cmp = icmp sge i32 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_sge_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_sge_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, gt -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_sge_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, gt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_sge_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-61439 // =0xffffffffffff1001 +; CHECK-GI-NEXT: movk x8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, ge +; CHECK-GI-NEXT: ret %cmp = icmp sge i64 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_uge(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_uge: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, hi -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_uge: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_uge: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #4097 // =0x1001 +; CHECK-GI-NEXT: movk w8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, hs +; CHECK-GI-NEXT: ret %cmp = icmp uge i32 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_uge_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_uge_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, hi -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_uge_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, hi +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_uge_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-61439 // =0xffffffffffff1001 +; CHECK-GI-NEXT: movk x8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, hs +; CHECK-GI-NEXT: ret %cmp = icmp uge i64 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_ult(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_ult: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, ls -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_ult: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_ult: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #4097 // =0x1001 +; CHECK-GI-NEXT: movk w8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, lo +; CHECK-GI-NEXT: ret %cmp = icmp ult i32 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_ult_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_ult_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4079, lsl #12 // =16707584 -; CHECK-NEXT: cset w0, ls -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_ult_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4079, lsl #12 // =16707584 +; CHECK-SD-NEXT: cset w0, ls +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_ult_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-61439 // =0xffffffffffff1001 +; CHECK-GI-NEXT: movk x8, #65281, lsl #16 +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, lo +; CHECK-GI-NEXT: ret %cmp = icmp ult i64 %x, -16707583 ret i1 %cmp } define i1 @almost_immediate_neg_sle(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_sle: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, lt -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_sle: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_sle: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #-16773121 // =0xff000fff +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, le +; CHECK-GI-NEXT: ret %cmp = icmp sle i32 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_sle_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_sle_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, lt -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_sle_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_sle_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-16773121 // =0xffffffffff000fff +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, le +; CHECK-GI-NEXT: ret %cmp = icmp sle i64 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_sgt(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_sgt: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, ge -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_sgt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_sgt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #-16773121 // =0xff000fff +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, gt +; CHECK-GI-NEXT: ret %cmp = icmp sgt i32 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_sgt_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_sgt_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, ge -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_sgt_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, ge +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_sgt_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-16773121 // =0xffffffffff000fff +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, gt +; CHECK-GI-NEXT: ret %cmp = icmp sgt i64 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_ule(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_ule: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, lo -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_ule: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_ule: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #-16773121 // =0xff000fff +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, ls +; CHECK-GI-NEXT: ret %cmp = icmp ule i32 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_ule_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_ule_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, lo -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_ule_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, lo +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_ule_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-16773121 // =0xffffffffff000fff +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, ls +; CHECK-GI-NEXT: ret %cmp = icmp ule i64 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_ugt(i32 %x) { -; CHECK-LABEL: almost_immediate_neg_ugt: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn w0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, hs -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_ugt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_ugt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #-16773121 // =0xff000fff +; CHECK-GI-NEXT: cmp w0, w8 +; CHECK-GI-NEXT: cset w0, hi +; CHECK-GI-NEXT: ret %cmp = icmp ugt i32 %x, -16773121 ret i1 %cmp } define i1 @almost_immediate_neg_ugt_64(i64 %x) { -; CHECK-LABEL: almost_immediate_neg_ugt_64: -; CHECK: // %bb.0: -; CHECK-NEXT: cmn x0, #4095, lsl #12 // =16773120 -; CHECK-NEXT: cset w0, hs -; CHECK-NEXT: ret +; CHECK-SD-LABEL: almost_immediate_neg_ugt_64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn x0, #4095, lsl #12 // =16773120 +; CHECK-SD-NEXT: cset w0, hs +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: almost_immediate_neg_ugt_64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-16773121 // =0xffffffffff000fff +; CHECK-GI-NEXT: cmp x0, x8 +; CHECK-GI-NEXT: cset w0, hi +; CHECK-GI-NEXT: ret %cmp = icmp ugt i64 %x, -16773121 ret i1 %cmp } @@ -637,6 +814,24 @@ define i1 @cmn_nsw_neg(i32 %a, i32 %b) { ret i1 %cmp } +define i1 @cmn_swap(i32 %a, i32 %b) { +; CHECK-SD-LABEL: cmn_swap: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmn w0, w1 +; CHECK-SD-NEXT: cset w0, lt +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmn_swap: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: cmn w1, w0 +; CHECK-GI-NEXT: cset w0, lt +; CHECK-GI-NEXT: ret + %sub = sub nsw i32 0, %b + %cmp = icmp sgt i32 %sub, %a + ret i1 %cmp +} + + define i1 @cmn_nsw_neg_64(i64 %a, i64 %b) { ; CHECK-LABEL: cmn_nsw_neg_64: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll index 880bd29..d67aa08 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-predicated-scalable.ll @@ -14,20 +14,19 @@ target triple = "aarch64" define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: mov w8, #100 // =0x64 -; CHECK-NEXT: cntd x9 ; CHECK-NEXT: whilelo p1.d, xzr, x8 +; CHECK-NEXT: cntd x9 ; CHECK-NEXT: rdvl x10, #2 -; CHECK-NEXT: mov x11, x9 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: zip2 z0.d, z1.d, z1.d -; CHECK-NEXT: zip1 z1.d, z1.d, z1.d +; CHECK-NEXT: mov x11, x9 ; CHECK-NEXT: .LBB0_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: zip2 p2.d, p1.d, p1.d -; CHECK-NEXT: mov z6.d, z1.d -; CHECK-NEXT: mov z7.d, z0.d +; CHECK-NEXT: mov z6.d, z0.d +; CHECK-NEXT: mov z7.d, z1.d ; CHECK-NEXT: zip1 p1.d, p1.d, p1.d ; CHECK-NEXT: ld1d { z2.d }, p2/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1d { z4.d }, p2/z, [x1, #1, mul vl] @@ -39,14 +38,14 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p2/m, z7.d -; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: mov z1.d, p2/m, z7.d +; CHECK-NEXT: mov z0.d, p1/m, z6.d ; CHECK-NEXT: whilelo p1.d, x11, x8 ; CHECK-NEXT: add x11, x11, x9 ; CHECK-NEXT: b.mi .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit.block -; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d -; CHECK-NEXT: uzp2 z1.d, z1.d, z0.d +; CHECK-NEXT: uzp1 z2.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z0.d, z1.d ; CHECK-NEXT: faddv d0, p0, z2.d ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -111,21 +110,20 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr %cond) { ; CHECK-LABEL: complex_mul_predicated_v2f64: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: mov w11, #100 // =0x64 ; CHECK-NEXT: neg x10, x9 +; CHECK-NEXT: mov w11, #100 // =0x64 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x10, x11 ; CHECK-NEXT: rdvl x11, #2 -; CHECK-NEXT: zip2 z0.d, z1.d, z1.d -; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB1_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ld1w { z2.d }, p0/z, [x2, x8, lsl #2] -; CHECK-NEXT: mov z6.d, z1.d -; CHECK-NEXT: mov z7.d, z0.d +; CHECK-NEXT: mov z6.d, z0.d +; CHECK-NEXT: mov z7.d, z1.d ; CHECK-NEXT: add x8, x8, x9 ; CHECK-NEXT: cmpne p1.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmp x10, x8 @@ -141,12 +139,12 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr % ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p2/m, z7.d -; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: mov z1.d, p2/m, z7.d +; CHECK-NEXT: mov z0.d, p1/m, z6.d ; CHECK-NEXT: b.ne .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit.block -; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d -; CHECK-NEXT: uzp2 z1.d, z1.d, z0.d +; CHECK-NEXT: uzp1 z2.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z0.d, z1.d ; CHECK-NEXT: faddv d0, p0, z2.d ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -213,21 +211,20 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, ptr %cond) { ; CHECK-LABEL: complex_mul_predicated_x2_v2f64: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: mov w8, #100 // =0x64 -; CHECK-NEXT: cntd x9 ; CHECK-NEXT: whilelo p1.d, xzr, x8 +; CHECK-NEXT: cntd x9 ; CHECK-NEXT: rdvl x10, #2 -; CHECK-NEXT: cnth x11 ; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: cnth x11 ; CHECK-NEXT: mov x12, x9 -; CHECK-NEXT: zip2 z0.d, z1.d, z1.d -; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB2_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ld1w { z2.d }, p1/z, [x2] -; CHECK-NEXT: mov z6.d, z1.d -; CHECK-NEXT: mov z7.d, z0.d +; CHECK-NEXT: mov z6.d, z0.d +; CHECK-NEXT: mov z7.d, z1.d ; CHECK-NEXT: add x2, x2, x11 ; CHECK-NEXT: and z2.d, z2.d, #0xffffffff ; CHECK-NEXT: cmpne p1.d, p1/z, z2.d, #0 @@ -243,14 +240,14 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p2/m, z7.d -; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: mov z1.d, p2/m, z7.d +; CHECK-NEXT: mov z0.d, p1/m, z6.d ; CHECK-NEXT: whilelo p1.d, x12, x8 ; CHECK-NEXT: add x12, x12, x9 ; CHECK-NEXT: b.mi .LBB2_1 ; CHECK-NEXT: // %bb.2: // %exit.block -; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d -; CHECK-NEXT: uzp2 z1.d, z1.d, z0.d +; CHECK-NEXT: uzp1 z2.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z0.d, z1.d ; CHECK-NEXT: faddv d0, p0, z2.d ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll index 29be231..0646ca4 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll @@ -14,15 +14,14 @@ target triple = "aarch64" define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cntd x8 -; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: neg x9, x8 +; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and x9, x9, x10 ; CHECK-NEXT: rdvl x10, #2 -; CHECK-NEXT: zip2 z0.d, z1.d, z1.d -; CHECK-NEXT: zip1 z1.d, z1.d, z1.d ; CHECK-NEXT: .LBB0_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr z2, [x0, #1, mul vl] @@ -32,14 +31,14 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-NEXT: ldr z5, [x1] ; CHECK-NEXT: add x1, x1, x10 ; CHECK-NEXT: add x0, x0, x10 -; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #0 -; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #0 +; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #90 +; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: b.ne .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit.block -; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d -; CHECK-NEXT: uzp2 z1.d, z1.d, z0.d +; CHECK-NEXT: uzp1 z2.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z0.d, z1.d ; CHECK-NEXT: faddv d0, p0, z2.d ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -183,17 +182,16 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64_unrolled: ; CHECK: // %bb.0: // %entry +; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: mov w10, #1000 // =0x3e8 +; CHECK-NEXT: movi v2.2d, #0000000000000000 +; CHECK-NEXT: movi v3.2d, #0000000000000000 ; CHECK-NEXT: neg x9, x8 +; CHECK-NEXT: mov w10, #1000 // =0x3e8 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and x9, x9, x10 ; CHECK-NEXT: rdvl x10, #4 -; CHECK-NEXT: zip2 z0.d, z1.d, z1.d -; CHECK-NEXT: zip1 z1.d, z1.d, z1.d -; CHECK-NEXT: mov z2.d, z1.d -; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: .LBB2_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr z4, [x0, #1, mul vl] @@ -207,20 +205,20 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-NEXT: ldr z18, [x1, #3, mul vl] ; CHECK-NEXT: ldr z19, [x1, #2, mul vl] ; CHECK-NEXT: add x1, x1, x10 -; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z16.d, z5.d, #0 +; CHECK-NEXT: fcmla z1.d, p0/m, z7.d, z4.d, #0 ; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #0 ; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #0 -; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z16.d, z5.d, #90 +; CHECK-NEXT: fcmla z1.d, p0/m, z7.d, z4.d, #90 ; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #90 ; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #90 ; CHECK-NEXT: b.ne .LBB2_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z4.d, z2.d, z3.d -; CHECK-NEXT: uzp1 z5.d, z1.d, z0.d +; CHECK-NEXT: uzp1 z5.d, z0.d, z1.d ; CHECK-NEXT: uzp2 z2.d, z2.d, z3.d -; CHECK-NEXT: uzp2 z0.d, z1.d, z0.d +; CHECK-NEXT: uzp2 z0.d, z0.d, z1.d ; CHECK-NEXT: fadd z1.d, z4.d, z5.d ; CHECK-NEXT: fadd z2.d, z2.d, z0.d ; CHECK-NEXT: faddv d0, p0, z1.d @@ -310,15 +308,15 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia ; CHECK-LABEL: reduction_mix: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.2d, #0000000000000000 +; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: mov w11, #100 // =0x64 +; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: neg x10, x9 +; CHECK-NEXT: mov w11, #100 // =0x64 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x10, x11 ; CHECK-NEXT: rdvl x11, #2 -; CHECK-NEXT: zip2 z0.d, z2.d, z2.d -; CHECK-NEXT: zip1 z1.d, z2.d, z2.d ; CHECK-NEXT: .LBB3_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr z3, [x0] @@ -327,13 +325,13 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia ; CHECK-NEXT: ld1w { z5.d }, p0/z, [x3, x8, lsl #2] ; CHECK-NEXT: add x8, x8, x9 ; CHECK-NEXT: cmp x10, x8 -; CHECK-NEXT: fadd z0.d, z4.d, z0.d -; CHECK-NEXT: fadd z1.d, z3.d, z1.d +; CHECK-NEXT: fadd z1.d, z4.d, z1.d +; CHECK-NEXT: fadd z0.d, z3.d, z0.d ; CHECK-NEXT: add z2.d, z5.d, z2.d ; CHECK-NEXT: b.ne .LBB3_1 ; CHECK-NEXT: // %bb.2: // %middle.block -; CHECK-NEXT: uzp2 z3.d, z1.d, z0.d -; CHECK-NEXT: uzp1 z1.d, z1.d, z0.d +; CHECK-NEXT: uzp2 z3.d, z0.d, z1.d +; CHECK-NEXT: uzp1 z1.d, z0.d, z1.d ; CHECK-NEXT: uaddv d2, p0, z2.d ; CHECK-NEXT: faddv d0, p0, z3.d ; CHECK-NEXT: faddv d1, p0, z1.d diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll index a9618fd..05ecc9e 100644 --- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll +++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll @@ -131,18 +131,83 @@ define <4 x i64> @interleave2_v4i64(<2 x i64> %vec0, <2 x i64> %vec1) { ret <4 x i64> %retval } +define <4 x i16> @interleave2_same_const_splat_v4i16() { +; CHECK-SD-LABEL: interleave2_same_const_splat_v4i16: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: movi v0.4h, #3 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: interleave2_same_const_splat_v4i16: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #3 // =0x3 +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: mov v0.h[1], w8 +; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v0.4h +; CHECK-GI-NEXT: ret + %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> splat(i16 3), <2 x i16> splat(i16 3)) + ret <4 x i16> %retval +} + +define <4 x i16> @interleave2_diff_const_splat_v4i16() { +; CHECK-SD-LABEL: interleave2_diff_const_splat_v4i16: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: adrp x8, .LCPI11_0 +; CHECK-SD-NEXT: ldr d0, [x8, :lo12:.LCPI11_0] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: interleave2_diff_const_splat_v4i16: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #3 // =0x3 +; CHECK-GI-NEXT: mov w9, #4 // =0x4 +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: fmov s1, w9 +; CHECK-GI-NEXT: mov v0.h[1], w8 +; CHECK-GI-NEXT: mov v1.h[1], w9 +; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: ret + %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> splat(i16 3), <2 x i16> splat(i16 4)) + ret <4 x i16> %retval +} -; Float declarations -declare <4 x half> @llvm.vector.interleave2.v4f16(<2 x half>, <2 x half>) -declare <8 x half> @llvm.vector.interleave2.v8f16(<4 x half>, <4 x half>) -declare <16 x half> @llvm.vector.interleave2.v16f16(<8 x half>, <8 x half>) -declare <4 x float> @llvm.vector.interleave2.v4f32(<2 x float>, <2 x float>) -declare <8 x float> @llvm.vector.interleave2.v8f32(<4 x float>, <4 x float>) -declare <4 x double> @llvm.vector.interleave2.v4f64(<2 x double>, <2 x double>) - -; Integer declarations -declare <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8>, <16 x i8>) -declare <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16>, <8 x i16>) -declare <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32>, <4 x i32>) -declare <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64>, <2 x i64>) +define <4 x i16> @interleave2_same_nonconst_splat_v4i16(i16 %a) { +; CHECK-SD-LABEL: interleave2_same_nonconst_splat_v4i16: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: dup v0.4h, w0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: interleave2_same_nonconst_splat_v4i16: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: dup v0.4h, w0 +; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v0.4h +; CHECK-GI-NEXT: ret + %ins = insertelement <2 x i16> poison, i16 %a, i32 0 + %splat = shufflevector <2 x i16> %ins, <2 x i16> poison, <2 x i32> <i32 0, i32 0> + %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> %splat, <2 x i16> %splat) + ret <4 x i16> %retval +} + +define <4 x i16> @interleave2_diff_nonconst_splat_v4i16(i16 %a, i16 %b) { +; CHECK-SD-LABEL: interleave2_diff_nonconst_splat_v4i16: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: mov v0.h[1], w0 +; CHECK-SD-NEXT: mov v0.h[2], w1 +; CHECK-SD-NEXT: mov v0.h[3], w1 +; CHECK-SD-NEXT: rev32 v1.4h, v0.4h +; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: interleave2_diff_nonconst_splat_v4i16: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: dup v0.4h, w0 +; CHECK-GI-NEXT: dup v1.4h, w1 +; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: ret + %ins1 = insertelement <2 x i16> poison, i16 %a, i32 0 + %splat1 = shufflevector <2 x i16> %ins1, <2 x i16> poison, <2 x i32> <i32 0, i32 0> + %ins2 = insertelement <2 x i16> poison, i16 %b, i32 0 + %splat2 = shufflevector <2 x i16> %ins2, <2 x i16> poison, <2 x i32> <i32 0, i32 0> + %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> %splat1, <2 x i16> %splat2) + ret <4 x i16> %retval +} diff --git a/llvm/test/CodeGen/AArch64/late-taildup-computed-goto.ll b/llvm/test/CodeGen/AArch64/late-taildup-computed-goto.ll index c4a027c..381904f 100644 --- a/llvm/test/CodeGen/AArch64/late-taildup-computed-goto.ll +++ b/llvm/test/CodeGen/AArch64/late-taildup-computed-goto.ll @@ -25,77 +25,58 @@ define void @test_interp(ptr %frame, ptr %dst) { ; CHECK-NEXT: adrp x21, _opcode.targets@PAGE ; CHECK-NEXT: Lloh1: ; CHECK-NEXT: add x21, x21, _opcode.targets@PAGEOFF -; CHECK-NEXT: mov x22, xzr +; CHECK-NEXT: mov x24, xzr ; CHECK-NEXT: add x8, x21, xzr, lsl #3 ; CHECK-NEXT: mov x19, x1 ; CHECK-NEXT: mov x20, x0 -; CHECK-NEXT: add x23, x22, #1 +; CHECK-NEXT: mov x23, xzr +; CHECK-NEXT: mov w22, #1 ; =0x1 +; CHECK-NEXT: add x24, x24, #1 ; CHECK-NEXT: br x8 ; CHECK-NEXT: Ltmp0: ; Block address taken ; CHECK-NEXT: LBB0_1: ; %loop.header ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x8, x21, x23, lsl #3 +; CHECK-NEXT: add x8, x21, x24, lsl #3 ; CHECK-NEXT: mov x20, xzr -; CHECK-NEXT: mov x22, xzr -; CHECK-NEXT: add x23, x23, #1 +; CHECK-NEXT: mov x23, xzr +; CHECK-NEXT: add x24, x24, #1 ; CHECK-NEXT: br x8 ; CHECK-NEXT: Ltmp1: ; Block address taken ; CHECK-NEXT: LBB0_2: ; %op1.bb -; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: str xzr, [x19] -; CHECK-NEXT: mov w8, #1 ; =0x1 +; CHECK-NEXT: Ltmp2: ; Block address taken +; CHECK-NEXT: LBB0_3: ; %op6.bb +; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr x0, [x20, #-8]! -; CHECK-NEXT: ldr x9, [x0, #8] -; CHECK-NEXT: str x8, [x0] -; CHECK-NEXT: ldr x8, [x9, #48] +; CHECK-NEXT: ldr x8, [x0, #8] +; CHECK-NEXT: str x22, [x0] +; CHECK-NEXT: ldr x8, [x8, #48] ; CHECK-NEXT: blr x8 -; CHECK-NEXT: add x8, x21, x23, lsl #3 -; CHECK-NEXT: add x23, x23, #1 +; CHECK-NEXT: add x8, x21, x24, lsl #3 +; CHECK-NEXT: add x24, x24, #1 ; CHECK-NEXT: br x8 -; CHECK-NEXT: Ltmp2: ; Block address taken -; CHECK-NEXT: LBB0_3: ; %op2.bb +; CHECK-NEXT: Ltmp3: ; Block address taken +; CHECK-NEXT: LBB0_4: ; %op2.bb ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add x8, x21, x23, lsl #3 +; CHECK-NEXT: add x8, x21, x24, lsl #3 ; CHECK-NEXT: mov x20, xzr -; CHECK-NEXT: add x23, x23, #1 -; CHECK-NEXT: str x22, [x19] -; CHECK-NEXT: mov x22, xzr +; CHECK-NEXT: str x23, [x19] +; CHECK-NEXT: mov x23, xzr +; CHECK-NEXT: add x24, x24, #1 ; CHECK-NEXT: br x8 -; CHECK-NEXT: Ltmp3: ; Block address taken -; CHECK-NEXT: LBB0_4: ; %op4.bb -; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: str x22, [x19] -; CHECK-NEXT: add x10, x21, x23, lsl #3 -; CHECK-NEXT: add x23, x23, #1 -; CHECK-NEXT: ldur x8, [x22, #12] -; CHECK-NEXT: ldur x9, [x20, #-8] -; CHECK-NEXT: add x22, x22, #20 -; CHECK-NEXT: stp x8, x9, [x20, #-8] -; CHECK-NEXT: add x20, x20, #8 -; CHECK-NEXT: br x10 ; CHECK-NEXT: Ltmp4: ; Block address taken -; CHECK-NEXT: LBB0_5: ; %op5.bb +; CHECK-NEXT: LBB0_5: ; %op4.bb +; CHECK-NEXT: Ltmp5: ; Block address taken +; CHECK-NEXT: LBB0_6: ; %op5.bb ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: str x22, [x19] -; CHECK-NEXT: add x10, x21, x23, lsl #3 -; CHECK-NEXT: add x23, x23, #1 -; CHECK-NEXT: ldur x8, [x22, #12] +; CHECK-NEXT: str x23, [x19] +; CHECK-NEXT: ldur x8, [x23, #12] ; CHECK-NEXT: ldur x9, [x20, #-8] -; CHECK-NEXT: add x22, x22, #20 +; CHECK-NEXT: add x23, x23, #20 ; CHECK-NEXT: stp x8, x9, [x20, #-8] +; CHECK-NEXT: add x8, x21, x24, lsl #3 ; CHECK-NEXT: add x20, x20, #8 -; CHECK-NEXT: br x10 -; CHECK-NEXT: Ltmp5: ; Block address taken -; CHECK-NEXT: LBB0_6: ; %op6.bb -; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr x0, [x20, #-8]! -; CHECK-NEXT: mov w8, #1 ; =0x1 -; CHECK-NEXT: ldr x9, [x0, #8] -; CHECK-NEXT: str x8, [x0] -; CHECK-NEXT: ldr x8, [x9, #48] -; CHECK-NEXT: blr x8 -; CHECK-NEXT: add x8, x21, x23, lsl #3 -; CHECK-NEXT: add x23, x23, #1 +; CHECK-NEXT: add x24, x24, #1 ; CHECK-NEXT: br x8 ; CHECK-NEXT: .loh AdrpAdd Lloh0, Lloh1 entry: diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll index 4f0c408..048e988 100644 --- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll +++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll @@ -28,46 +28,28 @@ define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, ; ; CHECK-GI-LABEL: test_udot_v4i8: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ldr w8, [x0] -; CHECK-GI-NEXT: ldr w9, [x1] +; CHECK-GI-NEXT: ldr w8, [x1] +; CHECK-GI-NEXT: ldr w9, [x0] ; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: uxtb w8, w8 -; CHECK-GI-NEXT: uxtb w9, w9 -; CHECK-GI-NEXT: mov b1, v0.b[1] -; CHECK-GI-NEXT: mov b3, v0.b[2] -; CHECK-GI-NEXT: mov b5, v2.b[2] -; CHECK-GI-NEXT: mov b4, v0.b[3] -; CHECK-GI-NEXT: mov b0, v2.b[1] -; CHECK-GI-NEXT: mov b6, v2.b[3] -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: fmov s1, w8 -; CHECK-GI-NEXT: fmov w13, s5 -; CHECK-GI-NEXT: fmov w8, s4 -; CHECK-GI-NEXT: fmov w12, s0 -; CHECK-GI-NEXT: uxtb w10, w10 -; CHECK-GI-NEXT: uxtb w11, w11 -; CHECK-GI-NEXT: uxtb w13, w13 -; CHECK-GI-NEXT: uxtb w8, w8 -; CHECK-GI-NEXT: uxtb w12, w12 -; CHECK-GI-NEXT: mov v1.h[1], w10 -; CHECK-GI-NEXT: fmov w10, s6 -; CHECK-GI-NEXT: fmov s0, w11 -; CHECK-GI-NEXT: fmov s3, w13 -; CHECK-GI-NEXT: mov v2.h[1], w12 -; CHECK-GI-NEXT: uxtb w10, w10 -; CHECK-GI-NEXT: mov v0.h[1], w8 -; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-NEXT: mov v3.h[1], w10 -; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 -; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 -; CHECK-GI-NEXT: mov v1.d[1], v0.d[0] -; CHECK-GI-NEXT: mov v2.d[1], v3.d[0] -; CHECK-GI-NEXT: mul v0.4s, v2.4s, v1.4s -; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov s1, w9 +; CHECK-GI-NEXT: mov b2, v0.b[1] +; CHECK-GI-NEXT: mov v3.b[0], v0.b[0] +; CHECK-GI-NEXT: mov b4, v1.b[1] +; CHECK-GI-NEXT: mov v5.b[0], v1.b[0] +; CHECK-GI-NEXT: mov v3.b[1], v2.b[0] +; CHECK-GI-NEXT: mov b2, v0.b[2] +; CHECK-GI-NEXT: mov b0, v0.b[3] +; CHECK-GI-NEXT: mov v5.b[1], v4.b[0] +; CHECK-GI-NEXT: mov b4, v1.b[2] +; CHECK-GI-NEXT: mov b1, v1.b[3] +; CHECK-GI-NEXT: mov v3.b[2], v2.b[0] +; CHECK-GI-NEXT: mov v5.b[2], v4.b[0] +; CHECK-GI-NEXT: mov v3.b[3], v0.b[0] +; CHECK-GI-NEXT: mov v5.b[3], v1.b[0] +; CHECK-GI-NEXT: ushll v0.8h, v3.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v5.8b, #0 +; CHECK-GI-NEXT: mul v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: uaddlv s0, v0.4h ; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: add w0, w8, w2 ; CHECK-GI-NEXT: ret @@ -128,46 +110,28 @@ define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, ; ; CHECK-GI-LABEL: test_sdot_v4i8: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ldr w8, [x0] -; CHECK-GI-NEXT: ldr w9, [x1] +; CHECK-GI-NEXT: ldr w8, [x1] +; CHECK-GI-NEXT: ldr w9, [x0] ; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov b1, v0.b[1] -; CHECK-GI-NEXT: mov b3, v0.b[2] -; CHECK-GI-NEXT: mov b5, v2.b[2] -; CHECK-GI-NEXT: mov b4, v0.b[3] -; CHECK-GI-NEXT: mov b0, v2.b[1] -; CHECK-GI-NEXT: mov b6, v2.b[3] -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s3 -; CHECK-GI-NEXT: fmov s1, w8 -; CHECK-GI-NEXT: fmov w13, s5 -; CHECK-GI-NEXT: fmov w8, s4 -; CHECK-GI-NEXT: fmov w12, s0 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v1.h[1], w10 -; CHECK-GI-NEXT: fmov w10, s6 -; CHECK-GI-NEXT: fmov s0, w11 -; CHECK-GI-NEXT: fmov s3, w13 -; CHECK-GI-NEXT: mov v2.h[1], w12 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v0.h[1], w8 -; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-GI-NEXT: mov v3.h[1], w10 -; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 -; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0 -; CHECK-GI-NEXT: mov v1.d[1], v0.d[0] -; CHECK-GI-NEXT: mov v2.d[1], v3.d[0] -; CHECK-GI-NEXT: mul v0.4s, v2.4s, v1.4s -; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov s1, w9 +; CHECK-GI-NEXT: mov b2, v0.b[1] +; CHECK-GI-NEXT: mov v3.b[0], v0.b[0] +; CHECK-GI-NEXT: mov b4, v1.b[1] +; CHECK-GI-NEXT: mov v5.b[0], v1.b[0] +; CHECK-GI-NEXT: mov v3.b[1], v2.b[0] +; CHECK-GI-NEXT: mov b2, v0.b[2] +; CHECK-GI-NEXT: mov b0, v0.b[3] +; CHECK-GI-NEXT: mov v5.b[1], v4.b[0] +; CHECK-GI-NEXT: mov b4, v1.b[2] +; CHECK-GI-NEXT: mov b1, v1.b[3] +; CHECK-GI-NEXT: mov v3.b[2], v2.b[0] +; CHECK-GI-NEXT: mov v5.b[2], v4.b[0] +; CHECK-GI-NEXT: mov v3.b[3], v0.b[0] +; CHECK-GI-NEXT: mov v5.b[3], v1.b[0] +; CHECK-GI-NEXT: sshll v0.8h, v3.8b, #0 +; CHECK-GI-NEXT: sshll v1.8h, v5.8b, #0 +; CHECK-GI-NEXT: mul v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: saddlv s0, v0.4h ; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: add w0, w8, w2 ; CHECK-GI-NEXT: ret @@ -205,22 +169,18 @@ define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8 ; ; CHECK-GI-LABEL: test_sdot_v4i8_double: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 -; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 -; CHECK-GI-NEXT: shl v0.4s, v0.4s, #24 -; CHECK-GI-NEXT: shl v1.4s, v1.4s, #24 -; CHECK-GI-NEXT: shl v2.4s, v2.4s, #24 -; CHECK-GI-NEXT: shl v3.4s, v3.4s, #24 -; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #24 -; CHECK-GI-NEXT: sshr v1.4s, v1.4s, #24 -; CHECK-GI-NEXT: sshr v2.4s, v2.4s, #24 -; CHECK-GI-NEXT: sshr v3.4s, v3.4s, #24 -; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s -; CHECK-GI-NEXT: mul v1.4s, v2.4s, v3.4s -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: shl v0.4h, v0.4h, #8 +; CHECK-GI-NEXT: shl v1.4h, v1.4h, #8 +; CHECK-GI-NEXT: shl v2.4h, v2.4h, #8 +; CHECK-GI-NEXT: shl v3.4h, v3.4h, #8 +; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #8 +; CHECK-GI-NEXT: sshr v1.4h, v1.4h, #8 +; CHECK-GI-NEXT: sshr v2.4h, v2.4h, #8 +; CHECK-GI-NEXT: sshr v3.4h, v3.4h, #8 +; CHECK-GI-NEXT: mul v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: mul v1.4h, v2.4h, v3.4h +; CHECK-GI-NEXT: saddlv s0, v0.4h +; CHECK-GI-NEXT: saddlv s1, v1.4h ; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: add w0, w8, w9 @@ -414,31 +374,60 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ldr d0, [x0] ; CHECK-GI-NEXT: ldr d1, [x1] -; CHECK-GI-NEXT: umov w8, v1.b[4] -; CHECK-GI-NEXT: umov w9, v0.b[4] -; CHECK-GI-NEXT: umov w10, v1.b[0] -; CHECK-GI-NEXT: umov w12, v0.b[0] -; CHECK-GI-NEXT: umov w11, v1.b[1] -; CHECK-GI-NEXT: umov w13, v0.b[1] -; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov s2, w10 -; CHECK-GI-NEXT: umov w9, v1.b[2] -; CHECK-GI-NEXT: fmov s3, w12 -; CHECK-GI-NEXT: umov w10, v1.b[3] -; CHECK-GI-NEXT: fmov s4, w8 -; CHECK-GI-NEXT: mov v2.s[1], w11 -; CHECK-GI-NEXT: umov w8, v0.b[2] -; CHECK-GI-NEXT: mov v3.s[1], w13 -; CHECK-GI-NEXT: umov w11, v0.b[3] -; CHECK-GI-NEXT: mov v4.s[1], wzr -; CHECK-GI-NEXT: mov v2.s[2], w9 -; CHECK-GI-NEXT: mov v3.s[2], w8 -; CHECK-GI-NEXT: mov v4.s[2], wzr -; CHECK-GI-NEXT: mov v2.s[3], w10 -; CHECK-GI-NEXT: mov v3.s[3], w11 -; CHECK-GI-NEXT: mov v4.s[3], wzr -; CHECK-GI-NEXT: mla v4.4s, v2.4s, v3.4s -; CHECK-GI-NEXT: addv s0, v4.4s +; CHECK-GI-NEXT: mov b2, v0.b[1] +; CHECK-GI-NEXT: mov b3, v1.b[1] +; CHECK-GI-NEXT: fmov w8, s1 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: mov b4, v1.b[2] +; CHECK-GI-NEXT: mov b5, v0.b[2] +; CHECK-GI-NEXT: mov b6, v0.b[3] +; CHECK-GI-NEXT: mov b7, v1.b[3] +; CHECK-GI-NEXT: mov b0, v0.b[4] +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: mov b1, v1.b[4] +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: fmov s2, w8 +; CHECK-GI-NEXT: fmov w8, s4 +; CHECK-GI-NEXT: fmov s3, w9 +; CHECK-GI-NEXT: fmov w9, s5 +; CHECK-GI-NEXT: uxtb w10, w10 +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: mov v2.h[1], w10 +; CHECK-GI-NEXT: mov v3.h[1], w11 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[2], w8 +; CHECK-GI-NEXT: mov v3.h[2], w9 +; CHECK-GI-NEXT: fmov w8, s7 +; CHECK-GI-NEXT: fmov w9, s6 +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[3], w8 +; CHECK-GI-NEXT: fmov w8, s1 +; CHECK-GI-NEXT: mov v3.h[3], w9 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[4], w8 +; CHECK-GI-NEXT: mov v3.h[4], w9 +; CHECK-GI-NEXT: mul v0.8h, v2.8h, v3.8h +; CHECK-GI-NEXT: umov w8, v0.h[0] +; CHECK-GI-NEXT: umov w9, v0.h[4] +; CHECK-GI-NEXT: umov w10, v0.h[1] +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: fmov s2, w9 +; CHECK-GI-NEXT: umov w8, v0.h[2] +; CHECK-GI-NEXT: umov w9, v0.h[3] +; CHECK-GI-NEXT: mov v1.s[1], w10 +; CHECK-GI-NEXT: mov v2.s[1], wzr +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: mov v2.s[2], wzr +; CHECK-GI-NEXT: mov v1.s[3], w9 +; CHECK-GI-NEXT: mov v2.s[3], wzr +; CHECK-GI-NEXT: add v0.4s, v1.4s, v2.4s +; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: add w0, w8, w2 ; CHECK-GI-NEXT: ret @@ -511,31 +500,60 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ldr d0, [x0] ; CHECK-GI-NEXT: ldr d1, [x1] -; CHECK-GI-NEXT: smov w8, v1.b[4] -; CHECK-GI-NEXT: smov w9, v0.b[4] -; CHECK-GI-NEXT: smov w10, v1.b[0] -; CHECK-GI-NEXT: smov w12, v0.b[0] -; CHECK-GI-NEXT: smov w11, v1.b[1] -; CHECK-GI-NEXT: smov w13, v0.b[1] -; CHECK-GI-NEXT: mul w8, w8, w9 -; CHECK-GI-NEXT: fmov s2, w10 -; CHECK-GI-NEXT: smov w9, v1.b[2] -; CHECK-GI-NEXT: fmov s3, w12 -; CHECK-GI-NEXT: smov w10, v1.b[3] -; CHECK-GI-NEXT: fmov s4, w8 -; CHECK-GI-NEXT: mov v2.s[1], w11 -; CHECK-GI-NEXT: smov w8, v0.b[2] -; CHECK-GI-NEXT: mov v3.s[1], w13 -; CHECK-GI-NEXT: smov w11, v0.b[3] -; CHECK-GI-NEXT: mov v4.s[1], wzr -; CHECK-GI-NEXT: mov v2.s[2], w9 -; CHECK-GI-NEXT: mov v3.s[2], w8 -; CHECK-GI-NEXT: mov v4.s[2], wzr -; CHECK-GI-NEXT: mov v2.s[3], w10 -; CHECK-GI-NEXT: mov v3.s[3], w11 -; CHECK-GI-NEXT: mov v4.s[3], wzr -; CHECK-GI-NEXT: mla v4.4s, v2.4s, v3.4s -; CHECK-GI-NEXT: addv s0, v4.4s +; CHECK-GI-NEXT: mov b2, v0.b[1] +; CHECK-GI-NEXT: mov b3, v1.b[1] +; CHECK-GI-NEXT: fmov w8, s1 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: mov b4, v1.b[2] +; CHECK-GI-NEXT: mov b5, v0.b[2] +; CHECK-GI-NEXT: mov b6, v0.b[3] +; CHECK-GI-NEXT: mov b7, v1.b[3] +; CHECK-GI-NEXT: mov b0, v0.b[4] +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov b1, v1.b[4] +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: fmov s2, w8 +; CHECK-GI-NEXT: fmov w8, s4 +; CHECK-GI-NEXT: fmov s3, w9 +; CHECK-GI-NEXT: fmov w9, s5 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov v2.h[1], w10 +; CHECK-GI-NEXT: mov v3.h[1], w11 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[2], w8 +; CHECK-GI-NEXT: mov v3.h[2], w9 +; CHECK-GI-NEXT: fmov w8, s7 +; CHECK-GI-NEXT: fmov w9, s6 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[3], w8 +; CHECK-GI-NEXT: fmov w8, s1 +; CHECK-GI-NEXT: mov v3.h[3], w9 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[4], w8 +; CHECK-GI-NEXT: mov v3.h[4], w9 +; CHECK-GI-NEXT: mul v0.8h, v2.8h, v3.8h +; CHECK-GI-NEXT: smov w8, v0.h[0] +; CHECK-GI-NEXT: smov w9, v0.h[4] +; CHECK-GI-NEXT: smov w10, v0.h[1] +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: fmov s2, w9 +; CHECK-GI-NEXT: smov w8, v0.h[2] +; CHECK-GI-NEXT: smov w9, v0.h[3] +; CHECK-GI-NEXT: mov v1.s[1], w10 +; CHECK-GI-NEXT: mov v2.s[1], wzr +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: mov v2.s[2], wzr +; CHECK-GI-NEXT: mov v1.s[3], w9 +; CHECK-GI-NEXT: mov v2.s[3], wzr +; CHECK-GI-NEXT: add v0.4s, v1.4s, v2.4s +; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: add w0, w8, w2 ; CHECK-GI-NEXT: ret @@ -571,59 +589,117 @@ define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8 ; CHECK-GI-LABEL: test_sdot_v5i8_double: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-NEXT: mov b17, v0.b[1] +; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 ; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2 ; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q3 -; CHECK-GI-NEXT: smov w9, v1.b[0] -; CHECK-GI-NEXT: smov w10, v0.b[4] -; CHECK-GI-NEXT: smov w11, v1.b[4] -; CHECK-GI-NEXT: smov w12, v2.b[0] -; CHECK-GI-NEXT: smov w13, v2.b[4] -; CHECK-GI-NEXT: smov w14, v3.b[4] -; CHECK-GI-NEXT: smov w8, v0.b[0] -; CHECK-GI-NEXT: smov w16, v3.b[0] -; CHECK-GI-NEXT: smov w15, v0.b[1] -; CHECK-GI-NEXT: fmov s5, w9 -; CHECK-GI-NEXT: mul w9, w10, w11 -; CHECK-GI-NEXT: smov w10, v1.b[1] -; CHECK-GI-NEXT: fmov s6, w12 -; CHECK-GI-NEXT: mul w12, w13, w14 -; CHECK-GI-NEXT: smov w11, v2.b[1] -; CHECK-GI-NEXT: smov w13, v3.b[1] -; CHECK-GI-NEXT: fmov s4, w8 -; CHECK-GI-NEXT: fmov s7, w16 -; CHECK-GI-NEXT: fmov s16, w9 -; CHECK-GI-NEXT: smov w8, v0.b[2] -; CHECK-GI-NEXT: smov w14, v1.b[2] -; CHECK-GI-NEXT: fmov s17, w12 -; CHECK-GI-NEXT: smov w9, v3.b[2] -; CHECK-GI-NEXT: mov v5.s[1], w10 -; CHECK-GI-NEXT: mov v4.s[1], w15 -; CHECK-GI-NEXT: smov w15, v2.b[2] -; CHECK-GI-NEXT: mov v6.s[1], w11 -; CHECK-GI-NEXT: mov v16.s[1], wzr -; CHECK-GI-NEXT: mov v7.s[1], w13 -; CHECK-GI-NEXT: smov w10, v0.b[3] -; CHECK-GI-NEXT: mov v17.s[1], wzr -; CHECK-GI-NEXT: smov w11, v1.b[3] -; CHECK-GI-NEXT: smov w12, v2.b[3] -; CHECK-GI-NEXT: smov w13, v3.b[3] -; CHECK-GI-NEXT: mov v5.s[2], w14 -; CHECK-GI-NEXT: mov v4.s[2], w8 -; CHECK-GI-NEXT: mov v6.s[2], w15 -; CHECK-GI-NEXT: mov v16.s[2], wzr -; CHECK-GI-NEXT: mov v7.s[2], w9 -; CHECK-GI-NEXT: mov v17.s[2], wzr -; CHECK-GI-NEXT: mov v5.s[3], w11 -; CHECK-GI-NEXT: mov v4.s[3], w10 -; CHECK-GI-NEXT: mov v6.s[3], w12 -; CHECK-GI-NEXT: mov v16.s[3], wzr -; CHECK-GI-NEXT: mov v7.s[3], w13 -; CHECK-GI-NEXT: mov v17.s[3], wzr -; CHECK-GI-NEXT: mla v16.4s, v4.4s, v5.4s -; CHECK-GI-NEXT: mla v17.4s, v6.4s, v7.4s -; CHECK-GI-NEXT: addv s0, v16.4s -; CHECK-GI-NEXT: addv s1, v17.4s +; CHECK-GI-NEXT: fmov w11, s1 +; CHECK-GI-NEXT: mov b25, v1.b[1] +; CHECK-GI-NEXT: mov b16, v1.b[2] +; CHECK-GI-NEXT: mov b7, v1.b[3] +; CHECK-GI-NEXT: mov b5, v1.b[4] +; CHECK-GI-NEXT: mov b22, v2.b[1] +; CHECK-GI-NEXT: mov b23, v3.b[1] +; CHECK-GI-NEXT: sxtb w9, w8 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: mov b24, v0.b[2] +; CHECK-GI-NEXT: fmov w8, s17 +; CHECK-GI-NEXT: mov b6, v0.b[3] +; CHECK-GI-NEXT: mov b4, v0.b[4] +; CHECK-GI-NEXT: fmov s1, w9 +; CHECK-GI-NEXT: mov b18, v2.b[2] +; CHECK-GI-NEXT: mov b19, v2.b[3] +; CHECK-GI-NEXT: mov b0, v2.b[4] +; CHECK-GI-NEXT: fmov w9, s25 +; CHECK-GI-NEXT: fmov w12, s22 +; CHECK-GI-NEXT: sxtb w10, w8 +; CHECK-GI-NEXT: mov b21, v3.b[2] +; CHECK-GI-NEXT: fmov w13, s23 +; CHECK-GI-NEXT: mov b20, v3.b[3] +; CHECK-GI-NEXT: mov b17, v3.b[4] +; CHECK-GI-NEXT: fmov w8, s24 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: sxtb w12, w12 +; CHECK-GI-NEXT: mov v1.h[1], w10 +; CHECK-GI-NEXT: sxtb w13, w13 +; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: fmov s2, w11 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v2.h[1], w9 +; CHECK-GI-NEXT: fmov w9, s16 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: mov v1.h[2], w8 +; CHECK-GI-NEXT: fmov w8, s7 +; CHECK-GI-NEXT: fmov s3, w10 +; CHECK-GI-NEXT: fmov w10, s18 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: fmov s22, w11 +; CHECK-GI-NEXT: fmov w11, s21 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov v3.h[1], w12 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v2.h[2], w9 +; CHECK-GI-NEXT: mov v22.h[1], w13 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: fmov w9, s19 +; CHECK-GI-NEXT: fmov w12, s6 +; CHECK-GI-NEXT: mov v3.h[2], w10 +; CHECK-GI-NEXT: fmov w10, s20 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v22.h[2], w11 +; CHECK-GI-NEXT: sxtb w12, w12 +; CHECK-GI-NEXT: fmov w11, s4 +; CHECK-GI-NEXT: mov v2.h[3], w8 +; CHECK-GI-NEXT: fmov w8, s5 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v1.h[3], w12 +; CHECK-GI-NEXT: mov v3.h[3], w9 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: mov v22.h[3], w10 +; CHECK-GI-NEXT: fmov w10, s17 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v1.h[4], w11 +; CHECK-GI-NEXT: mov v2.h[4], w8 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v3.h[4], w9 +; CHECK-GI-NEXT: mov v22.h[4], w10 +; CHECK-GI-NEXT: mul v0.8h, v1.8h, v2.8h +; CHECK-GI-NEXT: mul v1.8h, v3.8h, v22.8h +; CHECK-GI-NEXT: smov w8, v0.h[0] +; CHECK-GI-NEXT: smov w9, v0.h[4] +; CHECK-GI-NEXT: smov w11, v0.h[1] +; CHECK-GI-NEXT: smov w10, v1.h[0] +; CHECK-GI-NEXT: smov w12, v1.h[4] +; CHECK-GI-NEXT: smov w13, v1.h[1] +; CHECK-GI-NEXT: fmov s2, w8 +; CHECK-GI-NEXT: fmov s3, w9 +; CHECK-GI-NEXT: smov w8, v0.h[2] +; CHECK-GI-NEXT: smov w9, v1.h[2] +; CHECK-GI-NEXT: fmov s4, w10 +; CHECK-GI-NEXT: fmov s5, w12 +; CHECK-GI-NEXT: mov v2.s[1], w11 +; CHECK-GI-NEXT: mov v3.s[1], wzr +; CHECK-GI-NEXT: smov w10, v0.h[3] +; CHECK-GI-NEXT: smov w11, v1.h[3] +; CHECK-GI-NEXT: mov v4.s[1], w13 +; CHECK-GI-NEXT: mov v5.s[1], wzr +; CHECK-GI-NEXT: mov v2.s[2], w8 +; CHECK-GI-NEXT: mov v3.s[2], wzr +; CHECK-GI-NEXT: mov v4.s[2], w9 +; CHECK-GI-NEXT: mov v5.s[2], wzr +; CHECK-GI-NEXT: mov v2.s[3], w10 +; CHECK-GI-NEXT: mov v3.s[3], wzr +; CHECK-GI-NEXT: mov v4.s[3], w11 +; CHECK-GI-NEXT: mov v5.s[3], wzr +; CHECK-GI-NEXT: add v0.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v1.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s ; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: add w0, w8, w9 @@ -2303,11 +2379,14 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; ; CHECK-GI-LABEL: test_udot_v25i8: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: stp x26, x25, [sp, #-64]! // 16-byte Folded Spill -; CHECK-GI-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 64 +; CHECK-GI-NEXT: sub sp, sp, #112 +; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 112 ; CHECK-GI-NEXT: .cfi_offset w19, -8 ; CHECK-GI-NEXT: .cfi_offset w20, -16 ; CHECK-GI-NEXT: .cfi_offset w21, -24 @@ -2316,132 +2395,282 @@ define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; CHECK-GI-NEXT: .cfi_offset w24, -48 ; CHECK-GI-NEXT: .cfi_offset w25, -56 ; CHECK-GI-NEXT: .cfi_offset w26, -64 -; CHECK-GI-NEXT: ldp q1, q7, [x1] +; CHECK-GI-NEXT: .cfi_offset w27, -72 +; CHECK-GI-NEXT: .cfi_offset w28, -80 +; CHECK-GI-NEXT: .cfi_offset w30, -88 +; CHECK-GI-NEXT: .cfi_offset w29, -96 +; CHECK-GI-NEXT: ldp q2, q1, [x1] ; CHECK-GI-NEXT: fmov s0, wzr -; CHECK-GI-NEXT: ldp q16, q3, [x0] -; CHECK-GI-NEXT: umov w9, v1.b[4] -; CHECK-GI-NEXT: umov w11, v1.b[5] -; CHECK-GI-NEXT: umov w18, v1.b[0] -; CHECK-GI-NEXT: umov w0, v1.b[12] -; CHECK-GI-NEXT: umov w3, v7.b[4] -; CHECK-GI-NEXT: umov w12, v1.b[1] -; CHECK-GI-NEXT: umov w13, v1.b[6] -; CHECK-GI-NEXT: umov w1, v1.b[13] -; CHECK-GI-NEXT: umov w4, v7.b[5] -; CHECK-GI-NEXT: umov w15, v1.b[2] -; CHECK-GI-NEXT: umov w8, v1.b[3] -; CHECK-GI-NEXT: umov w16, v1.b[7] -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: umov w14, v1.b[8] -; CHECK-GI-NEXT: umov w17, v1.b[9] -; CHECK-GI-NEXT: umov w10, v1.b[10] -; CHECK-GI-NEXT: umov w9, v1.b[11] -; CHECK-GI-NEXT: umov w5, v1.b[14] -; CHECK-GI-NEXT: umov w6, v7.b[0] -; CHECK-GI-NEXT: fmov s4, w0 -; CHECK-GI-NEXT: fmov s5, w3 -; CHECK-GI-NEXT: mov v2.s[1], w11 -; CHECK-GI-NEXT: umov w11, v1.b[15] -; CHECK-GI-NEXT: fmov s1, w18 -; CHECK-GI-NEXT: umov w7, v7.b[1] -; CHECK-GI-NEXT: umov w18, v7.b[6] -; CHECK-GI-NEXT: umov w21, v16.b[4] -; CHECK-GI-NEXT: mov v4.s[1], w1 -; CHECK-GI-NEXT: mov v5.s[1], w4 -; CHECK-GI-NEXT: fmov s6, w14 -; CHECK-GI-NEXT: mov v1.s[1], w12 -; CHECK-GI-NEXT: umov w12, v7.b[3] -; CHECK-GI-NEXT: umov w14, v7.b[7] -; CHECK-GI-NEXT: mov v2.s[2], w13 -; CHECK-GI-NEXT: umov w13, v7.b[2] -; CHECK-GI-NEXT: umov w0, v7.b[8] -; CHECK-GI-NEXT: fmov s7, w6 -; CHECK-GI-NEXT: umov w23, v16.b[12] -; CHECK-GI-NEXT: umov w25, v3.b[4] -; CHECK-GI-NEXT: mov v6.s[1], w17 -; CHECK-GI-NEXT: mov v4.s[2], w5 -; CHECK-GI-NEXT: mov v5.s[2], w18 -; CHECK-GI-NEXT: mov v1.s[2], w15 -; CHECK-GI-NEXT: umov w6, v16.b[0] -; CHECK-GI-NEXT: umov w3, v16.b[1] -; CHECK-GI-NEXT: mov v2.s[3], w16 -; CHECK-GI-NEXT: mov v7.s[1], w7 -; CHECK-GI-NEXT: umov w16, v16.b[2] -; CHECK-GI-NEXT: umov w15, v16.b[3] -; CHECK-GI-NEXT: umov w22, v16.b[5] -; CHECK-GI-NEXT: umov w5, v16.b[6] -; CHECK-GI-NEXT: umov w18, v16.b[7] -; CHECK-GI-NEXT: umov w19, v16.b[8] -; CHECK-GI-NEXT: umov w7, v16.b[9] -; CHECK-GI-NEXT: umov w24, v16.b[13] -; CHECK-GI-NEXT: umov w1, v16.b[10] -; CHECK-GI-NEXT: umov w17, v16.b[11] -; CHECK-GI-NEXT: umov w20, v16.b[14] -; CHECK-GI-NEXT: umov w4, v16.b[15] -; CHECK-GI-NEXT: fmov s16, w21 -; CHECK-GI-NEXT: umov w21, v3.b[8] -; CHECK-GI-NEXT: umov w26, v3.b[5] -; CHECK-GI-NEXT: fmov s17, w23 -; CHECK-GI-NEXT: umov w23, v3.b[0] -; CHECK-GI-NEXT: fmov s18, w25 -; CHECK-GI-NEXT: umov w25, v3.b[3] -; CHECK-GI-NEXT: mov v16.s[1], w22 -; CHECK-GI-NEXT: umov w22, v3.b[1] -; CHECK-GI-NEXT: fmov s19, w6 -; CHECK-GI-NEXT: mov v17.s[1], w24 -; CHECK-GI-NEXT: umov w24, v3.b[2] -; CHECK-GI-NEXT: umov w6, v3.b[7] -; CHECK-GI-NEXT: mul w0, w0, w21 -; CHECK-GI-NEXT: mov v18.s[1], w26 -; CHECK-GI-NEXT: umov w26, v3.b[6] -; CHECK-GI-NEXT: fmov s3, w19 -; CHECK-GI-NEXT: fmov s20, w23 -; CHECK-GI-NEXT: mov v19.s[1], w3 -; CHECK-GI-NEXT: mov v16.s[2], w5 +; CHECK-GI-NEXT: str w2, [sp, #12] // 4-byte Folded Spill +; CHECK-GI-NEXT: mov b6, v2.b[3] +; CHECK-GI-NEXT: mov b7, v2.b[4] +; CHECK-GI-NEXT: mov b16, v2.b[5] +; CHECK-GI-NEXT: mov b19, v2.b[8] +; CHECK-GI-NEXT: mov b4, v2.b[1] +; CHECK-GI-NEXT: mov b5, v2.b[2] +; CHECK-GI-NEXT: mov b17, v2.b[6] +; CHECK-GI-NEXT: mov b18, v2.b[7] +; CHECK-GI-NEXT: mov b20, v2.b[9] +; CHECK-GI-NEXT: mov b21, v2.b[10] +; CHECK-GI-NEXT: mov b22, v2.b[11] +; CHECK-GI-NEXT: fmov w7, s2 +; CHECK-GI-NEXT: fmov w13, s6 +; CHECK-GI-NEXT: mov b6, v2.b[12] +; CHECK-GI-NEXT: fmov w2, s7 +; CHECK-GI-NEXT: mov b7, v2.b[13] +; CHECK-GI-NEXT: fmov w11, s16 +; CHECK-GI-NEXT: mov b16, v2.b[14] +; CHECK-GI-NEXT: mov b23, v2.b[15] +; CHECK-GI-NEXT: ldp q3, q2, [x0] +; CHECK-GI-NEXT: fmov w26, s19 +; CHECK-GI-NEXT: fmov w19, s4 +; CHECK-GI-NEXT: stp s17, s18, [sp, #4] // 8-byte Folded Spill +; CHECK-GI-NEXT: fmov w29, s5 +; CHECK-GI-NEXT: fmov w24, s20 +; CHECK-GI-NEXT: uxtb w8, w7 +; CHECK-GI-NEXT: mov b4, v3.b[2] +; CHECK-GI-NEXT: mov b5, v3.b[1] +; CHECK-GI-NEXT: uxtb w13, w13 +; CHECK-GI-NEXT: mov b17, v1.b[1] +; CHECK-GI-NEXT: fmov w22, s21 +; CHECK-GI-NEXT: uxtb w26, w26 +; CHECK-GI-NEXT: mov b18, v1.b[2] +; CHECK-GI-NEXT: fmov w18, s22 +; CHECK-GI-NEXT: uxtb w24, w24 +; CHECK-GI-NEXT: mov b19, v1.b[3] +; CHECK-GI-NEXT: fmov w16, s6 +; CHECK-GI-NEXT: uxtb w19, w19 +; CHECK-GI-NEXT: mov b21, v1.b[4] +; CHECK-GI-NEXT: fmov w15, s7 +; CHECK-GI-NEXT: uxtb w22, w22 +; CHECK-GI-NEXT: mov b7, v1.b[5] +; CHECK-GI-NEXT: mov b6, v3.b[3] +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: fmov w12, s23 +; CHECK-GI-NEXT: mov b22, v1.b[6] +; CHECK-GI-NEXT: mov b23, v1.b[7] +; CHECK-GI-NEXT: mov b20, v3.b[4] +; CHECK-GI-NEXT: fmov w28, s4 +; CHECK-GI-NEXT: fmov s4, w26 +; CHECK-GI-NEXT: fmov w14, s16 +; CHECK-GI-NEXT: fmov w27, s17 +; CHECK-GI-NEXT: fmov w5, s18 +; CHECK-GI-NEXT: uxtb w12, w12 +; CHECK-GI-NEXT: fmov w4, s19 +; CHECK-GI-NEXT: mov b19, v3.b[5] +; CHECK-GI-NEXT: uxtb w28, w28 +; CHECK-GI-NEXT: fmov w3, s21 +; CHECK-GI-NEXT: mov b18, v3.b[6] +; CHECK-GI-NEXT: uxtb w27, w27 +; CHECK-GI-NEXT: uxtb w5, w5 +; CHECK-GI-NEXT: fmov w1, s7 +; CHECK-GI-NEXT: mov b16, v3.b[7] +; CHECK-GI-NEXT: fmov w0, s22 +; CHECK-GI-NEXT: mov b17, v3.b[8] +; CHECK-GI-NEXT: fmov w17, s23 +; CHECK-GI-NEXT: mov b7, v3.b[9] +; CHECK-GI-NEXT: fmov w30, s5 +; CHECK-GI-NEXT: mov b5, v3.b[10] +; CHECK-GI-NEXT: mov b21, v3.b[11] +; CHECK-GI-NEXT: fmov w25, s6 +; CHECK-GI-NEXT: mov b6, v3.b[12] +; CHECK-GI-NEXT: fmov w23, s20 +; CHECK-GI-NEXT: mov b20, v3.b[13] +; CHECK-GI-NEXT: mov b22, v3.b[14] +; CHECK-GI-NEXT: fmov w6, s3 +; CHECK-GI-NEXT: mov b23, v3.b[15] +; CHECK-GI-NEXT: fmov s3, w8 +; CHECK-GI-NEXT: fmov w8, s1 +; CHECK-GI-NEXT: mov v4.h[1], w24 +; CHECK-GI-NEXT: fmov w21, s19 +; CHECK-GI-NEXT: mov b19, v2.b[1] +; CHECK-GI-NEXT: fmov w9, s17 +; CHECK-GI-NEXT: fmov w24, s6 +; CHECK-GI-NEXT: fmov w7, s16 +; CHECK-GI-NEXT: mov b16, v2.b[2] +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: mov v3.h[1], w19 +; CHECK-GI-NEXT: uxtb w19, w29 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: fmov w29, s5 +; CHECK-GI-NEXT: mov v4.h[2], w22 +; CHECK-GI-NEXT: uxtb w22, w6 +; CHECK-GI-NEXT: fmov s5, w8 +; CHECK-GI-NEXT: fmov w10, s7 +; CHECK-GI-NEXT: fmov s7, w9 +; CHECK-GI-NEXT: fmov w9, s16 +; CHECK-GI-NEXT: fmov w20, s18 +; CHECK-GI-NEXT: uxtb w29, w29 +; CHECK-GI-NEXT: fmov s6, w22 +; CHECK-GI-NEXT: fmov w22, s2 +; CHECK-GI-NEXT: uxtb w10, w10 +; CHECK-GI-NEXT: mov v5.h[1], w27 +; CHECK-GI-NEXT: uxtb w27, w30 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov b18, v2.b[3] +; CHECK-GI-NEXT: mov v3.h[2], w19 +; CHECK-GI-NEXT: uxtb w22, w22 +; CHECK-GI-NEXT: mov v6.h[1], w27 +; CHECK-GI-NEXT: fmov w27, s19 +; CHECK-GI-NEXT: mov v7.h[1], w10 +; CHECK-GI-NEXT: fmov w26, s21 +; CHECK-GI-NEXT: mov b17, v2.b[4] +; CHECK-GI-NEXT: fmov s16, w22 +; CHECK-GI-NEXT: mov v5.h[2], w5 +; CHECK-GI-NEXT: uxtb w5, w25 +; CHECK-GI-NEXT: uxtb w27, w27 +; CHECK-GI-NEXT: fmov w10, s18 +; CHECK-GI-NEXT: mov v3.h[3], w13 +; CHECK-GI-NEXT: uxtb w13, w4 +; CHECK-GI-NEXT: mov v6.h[2], w28 +; CHECK-GI-NEXT: fmov w8, s20 +; CHECK-GI-NEXT: mov v16.h[1], w27 +; CHECK-GI-NEXT: mov v7.h[2], w29 +; CHECK-GI-NEXT: mov b20, v2.b[5] +; CHECK-GI-NEXT: uxtb w10, w10 +; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: fmov w22, s17 +; CHECK-GI-NEXT: mov v5.h[3], w13 +; CHECK-GI-NEXT: uxtb w13, w2 +; CHECK-GI-NEXT: mov v6.h[3], w5 +; CHECK-GI-NEXT: mov b21, v2.b[6] +; CHECK-GI-NEXT: mov v16.h[2], w9 +; CHECK-GI-NEXT: uxtb w9, w18 +; CHECK-GI-NEXT: uxtb w18, w23 +; CHECK-GI-NEXT: mov v3.h[4], w13 +; CHECK-GI-NEXT: uxtb w13, w24 +; CHECK-GI-NEXT: fmov w27, s20 +; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v4.h[3], w9 +; CHECK-GI-NEXT: uxtb w9, w26 +; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v16.h[3], w10 +; CHECK-GI-NEXT: uxtb w10, w3 +; CHECK-GI-NEXT: mov v6.h[4], w18 +; CHECK-GI-NEXT: ldr w18, [sp, #4] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v7.h[3], w9 +; CHECK-GI-NEXT: uxtb w9, w16 +; CHECK-GI-NEXT: uxtb w16, w22 +; CHECK-GI-NEXT: mov v5.h[4], w10 +; CHECK-GI-NEXT: uxtb w10, w15 +; CHECK-GI-NEXT: uxtb w18, w18 +; CHECK-GI-NEXT: mov v4.h[4], w9 +; CHECK-GI-NEXT: uxtb w9, w21 +; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v16.h[4], w16 +; CHECK-GI-NEXT: mov v7.h[4], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #8] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v6.h[5], w9 +; CHECK-GI-NEXT: uxtb w9, w1 +; CHECK-GI-NEXT: mov v3.h[5], w11 +; CHECK-GI-NEXT: uxtb w11, w27 +; CHECK-GI-NEXT: fmov w19, s22 +; CHECK-GI-NEXT: fmov w28, s21 +; CHECK-GI-NEXT: uxtb w13, w13 +; CHECK-GI-NEXT: mov b17, v2.b[7] +; CHECK-GI-NEXT: mov v5.h[5], w9 +; CHECK-GI-NEXT: uxtb w9, w0 +; CHECK-GI-NEXT: mov v4.h[5], w10 +; CHECK-GI-NEXT: uxtb w10, w20 +; CHECK-GI-NEXT: mov v7.h[5], w8 +; CHECK-GI-NEXT: mov v16.h[5], w11 +; CHECK-GI-NEXT: uxtb w8, w14 +; CHECK-GI-NEXT: uxtb w11, w28 +; CHECK-GI-NEXT: mov v6.h[6], w10 +; CHECK-GI-NEXT: uxtb w10, w19 +; CHECK-GI-NEXT: fmov w6, s23 +; CHECK-GI-NEXT: mov v5.h[6], w9 +; CHECK-GI-NEXT: fmov w9, s17 +; CHECK-GI-NEXT: mov v3.h[6], w18 +; CHECK-GI-NEXT: mov v4.h[6], w8 +; CHECK-GI-NEXT: uxtb w8, w7 +; CHECK-GI-NEXT: mov v7.h[6], w10 +; CHECK-GI-NEXT: mov v16.h[6], w11 +; CHECK-GI-NEXT: uxtb w10, w6 ; CHECK-GI-NEXT: mov v0.s[1], wzr -; CHECK-GI-NEXT: mov v6.s[2], w10 -; CHECK-GI-NEXT: fmov s21, w0 -; CHECK-GI-NEXT: mov v17.s[2], w20 -; CHECK-GI-NEXT: mov v4.s[3], w11 -; CHECK-GI-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v3.s[1], w7 -; CHECK-GI-NEXT: mov v20.s[1], w22 -; CHECK-GI-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v18.s[2], w26 -; CHECK-GI-NEXT: mov v21.s[1], wzr -; CHECK-GI-NEXT: mov v16.s[3], w18 -; CHECK-GI-NEXT: mov v17.s[3], w4 -; CHECK-GI-NEXT: mov v7.s[2], w13 -; CHECK-GI-NEXT: mov v5.s[3], w14 -; CHECK-GI-NEXT: mov v19.s[2], w16 -; CHECK-GI-NEXT: mov v3.s[2], w1 +; CHECK-GI-NEXT: mov v6.h[7], w8 +; CHECK-GI-NEXT: uxtb w8, w17 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov v3.h[7], w13 +; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v4.h[7], w12 +; CHECK-GI-NEXT: mov v5.h[7], w8 +; CHECK-GI-NEXT: mov v7.h[7], w10 +; CHECK-GI-NEXT: mov v16.h[7], w9 +; CHECK-GI-NEXT: umov w8, v1.b[8] +; CHECK-GI-NEXT: umov w9, v2.b[8] ; CHECK-GI-NEXT: mov v0.s[2], wzr -; CHECK-GI-NEXT: mov v20.s[2], w24 -; CHECK-GI-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v18.s[3], w6 -; CHECK-GI-NEXT: mov v21.s[2], wzr -; CHECK-GI-NEXT: mul v2.4s, v2.4s, v16.4s -; CHECK-GI-NEXT: mul v4.4s, v4.4s, v17.4s -; CHECK-GI-NEXT: mov v1.s[3], w8 -; CHECK-GI-NEXT: mov v6.s[3], w9 -; CHECK-GI-NEXT: mov v7.s[3], w12 -; CHECK-GI-NEXT: mov v19.s[3], w15 -; CHECK-GI-NEXT: mov v3.s[3], w17 -; CHECK-GI-NEXT: mov v20.s[3], w25 +; CHECK-GI-NEXT: mul v3.8h, v3.8h, v6.8h +; CHECK-GI-NEXT: mul v2.8h, v4.8h, v7.8h +; CHECK-GI-NEXT: mul v1.8h, v5.8h, v16.8h +; CHECK-GI-NEXT: mul w15, w8, w9 ; CHECK-GI-NEXT: mov v0.s[3], wzr -; CHECK-GI-NEXT: mul v5.4s, v5.4s, v18.4s -; CHECK-GI-NEXT: mov v21.s[3], wzr -; CHECK-GI-NEXT: mla v2.4s, v1.4s, v19.4s -; CHECK-GI-NEXT: mla v4.4s, v6.4s, v3.4s -; CHECK-GI-NEXT: mla v5.4s, v7.4s, v20.4s -; CHECK-GI-NEXT: add v0.4s, v21.4s, v0.4s -; CHECK-GI-NEXT: add v1.4s, v2.4s, v4.4s -; CHECK-GI-NEXT: add v0.4s, v5.4s, v0.4s +; CHECK-GI-NEXT: umov w16, v3.h[0] +; CHECK-GI-NEXT: umov w18, v3.h[4] +; CHECK-GI-NEXT: umov w17, v3.h[1] +; CHECK-GI-NEXT: umov w1, v2.h[0] +; CHECK-GI-NEXT: umov w3, v2.h[4] +; CHECK-GI-NEXT: umov w0, v3.h[5] +; CHECK-GI-NEXT: umov w5, v1.h[0] +; CHECK-GI-NEXT: umov w7, v1.h[4] +; CHECK-GI-NEXT: umov w2, v2.h[1] +; CHECK-GI-NEXT: umov w4, v2.h[5] +; CHECK-GI-NEXT: umov w6, v1.h[1] +; CHECK-GI-NEXT: umov w19, v1.h[5] +; CHECK-GI-NEXT: umov w10, v3.h[2] +; CHECK-GI-NEXT: umov w8, v3.h[3] +; CHECK-GI-NEXT: umov w11, v3.h[6] +; CHECK-GI-NEXT: umov w9, v3.h[7] +; CHECK-GI-NEXT: fmov s3, w16 +; CHECK-GI-NEXT: fmov s4, w18 +; CHECK-GI-NEXT: fmov s5, w1 +; CHECK-GI-NEXT: fmov s6, w3 +; CHECK-GI-NEXT: fmov s7, w5 +; CHECK-GI-NEXT: fmov s16, w7 +; CHECK-GI-NEXT: fmov s17, w15 +; CHECK-GI-NEXT: umov w12, v2.h[2] +; CHECK-GI-NEXT: umov w13, v2.h[6] +; CHECK-GI-NEXT: umov w14, v1.h[2] +; CHECK-GI-NEXT: umov w16, v1.h[6] +; CHECK-GI-NEXT: mov v3.s[1], w17 +; CHECK-GI-NEXT: mov v4.s[1], w0 +; CHECK-GI-NEXT: mov v5.s[1], w2 +; CHECK-GI-NEXT: mov v6.s[1], w4 +; CHECK-GI-NEXT: mov v7.s[1], w6 +; CHECK-GI-NEXT: mov v16.s[1], w19 +; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v17.s[1], wzr +; CHECK-GI-NEXT: umov w15, v2.h[3] +; CHECK-GI-NEXT: umov w17, v2.h[7] +; CHECK-GI-NEXT: umov w18, v1.h[3] +; CHECK-GI-NEXT: umov w0, v1.h[7] +; CHECK-GI-NEXT: mov v3.s[2], w10 +; CHECK-GI-NEXT: mov v4.s[2], w11 +; CHECK-GI-NEXT: mov v5.s[2], w12 +; CHECK-GI-NEXT: mov v6.s[2], w13 +; CHECK-GI-NEXT: mov v7.s[2], w14 +; CHECK-GI-NEXT: mov v16.s[2], w16 +; CHECK-GI-NEXT: mov v17.s[2], wzr +; CHECK-GI-NEXT: mov v3.s[3], w8 +; CHECK-GI-NEXT: mov v4.s[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #12] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v5.s[3], w15 +; CHECK-GI-NEXT: mov v6.s[3], w17 +; CHECK-GI-NEXT: mov v7.s[3], w18 +; CHECK-GI-NEXT: mov v16.s[3], w0 +; CHECK-GI-NEXT: mov v17.s[3], wzr +; CHECK-GI-NEXT: add v1.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: add v2.4s, v5.4s, v6.4s +; CHECK-GI-NEXT: add v3.4s, v7.4s, v16.4s +; CHECK-GI-NEXT: add v0.4s, v17.4s, v0.4s +; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s ; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ldp x26, x25, [sp], #64 // 16-byte Folded Reload +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: add sp, sp, #112 ; CHECK-GI-NEXT: ret entry: %0 = load <25 x i8>, ptr %a @@ -2580,11 +2809,14 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; ; CHECK-GI-LABEL: test_sdot_v25i8: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: stp x26, x25, [sp, #-64]! // 16-byte Folded Spill -; CHECK-GI-NEXT: stp x24, x23, [sp, #16] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp x22, x21, [sp, #32] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 64 +; CHECK-GI-NEXT: sub sp, sp, #112 +; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 112 ; CHECK-GI-NEXT: .cfi_offset w19, -8 ; CHECK-GI-NEXT: .cfi_offset w20, -16 ; CHECK-GI-NEXT: .cfi_offset w21, -24 @@ -2593,132 +2825,283 @@ define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; CHECK-GI-NEXT: .cfi_offset w24, -48 ; CHECK-GI-NEXT: .cfi_offset w25, -56 ; CHECK-GI-NEXT: .cfi_offset w26, -64 -; CHECK-GI-NEXT: ldp q1, q7, [x1] +; CHECK-GI-NEXT: .cfi_offset w27, -72 +; CHECK-GI-NEXT: .cfi_offset w28, -80 +; CHECK-GI-NEXT: .cfi_offset w30, -88 +; CHECK-GI-NEXT: .cfi_offset w29, -96 +; CHECK-GI-NEXT: ldp q2, q1, [x1] ; CHECK-GI-NEXT: fmov s0, wzr -; CHECK-GI-NEXT: ldp q16, q3, [x0] -; CHECK-GI-NEXT: smov w9, v1.b[4] -; CHECK-GI-NEXT: smov w11, v1.b[5] -; CHECK-GI-NEXT: smov w18, v1.b[0] -; CHECK-GI-NEXT: smov w0, v1.b[12] -; CHECK-GI-NEXT: smov w3, v7.b[4] -; CHECK-GI-NEXT: smov w12, v1.b[1] -; CHECK-GI-NEXT: smov w13, v1.b[6] -; CHECK-GI-NEXT: smov w1, v1.b[13] -; CHECK-GI-NEXT: smov w4, v7.b[5] -; CHECK-GI-NEXT: smov w15, v1.b[2] -; CHECK-GI-NEXT: smov w8, v1.b[3] -; CHECK-GI-NEXT: smov w16, v1.b[7] -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: smov w14, v1.b[8] -; CHECK-GI-NEXT: smov w17, v1.b[9] -; CHECK-GI-NEXT: smov w10, v1.b[10] -; CHECK-GI-NEXT: smov w9, v1.b[11] -; CHECK-GI-NEXT: smov w5, v1.b[14] -; CHECK-GI-NEXT: smov w6, v7.b[0] -; CHECK-GI-NEXT: fmov s4, w0 -; CHECK-GI-NEXT: fmov s5, w3 -; CHECK-GI-NEXT: mov v2.s[1], w11 -; CHECK-GI-NEXT: smov w11, v1.b[15] -; CHECK-GI-NEXT: fmov s1, w18 -; CHECK-GI-NEXT: smov w7, v7.b[1] -; CHECK-GI-NEXT: smov w18, v7.b[6] -; CHECK-GI-NEXT: smov w21, v16.b[4] -; CHECK-GI-NEXT: mov v4.s[1], w1 -; CHECK-GI-NEXT: mov v5.s[1], w4 -; CHECK-GI-NEXT: fmov s6, w14 -; CHECK-GI-NEXT: mov v1.s[1], w12 -; CHECK-GI-NEXT: smov w12, v7.b[3] -; CHECK-GI-NEXT: smov w14, v7.b[7] -; CHECK-GI-NEXT: mov v2.s[2], w13 -; CHECK-GI-NEXT: smov w13, v7.b[2] -; CHECK-GI-NEXT: smov w0, v7.b[8] -; CHECK-GI-NEXT: fmov s7, w6 -; CHECK-GI-NEXT: smov w23, v16.b[12] -; CHECK-GI-NEXT: smov w25, v3.b[4] -; CHECK-GI-NEXT: mov v6.s[1], w17 -; CHECK-GI-NEXT: mov v4.s[2], w5 -; CHECK-GI-NEXT: mov v5.s[2], w18 -; CHECK-GI-NEXT: mov v1.s[2], w15 -; CHECK-GI-NEXT: smov w6, v16.b[0] -; CHECK-GI-NEXT: smov w3, v16.b[1] -; CHECK-GI-NEXT: mov v2.s[3], w16 -; CHECK-GI-NEXT: mov v7.s[1], w7 -; CHECK-GI-NEXT: smov w16, v16.b[2] -; CHECK-GI-NEXT: smov w15, v16.b[3] -; CHECK-GI-NEXT: smov w22, v16.b[5] -; CHECK-GI-NEXT: smov w5, v16.b[6] -; CHECK-GI-NEXT: smov w18, v16.b[7] -; CHECK-GI-NEXT: smov w19, v16.b[8] -; CHECK-GI-NEXT: smov w7, v16.b[9] -; CHECK-GI-NEXT: smov w24, v16.b[13] -; CHECK-GI-NEXT: smov w1, v16.b[10] -; CHECK-GI-NEXT: smov w17, v16.b[11] -; CHECK-GI-NEXT: smov w20, v16.b[14] -; CHECK-GI-NEXT: smov w4, v16.b[15] -; CHECK-GI-NEXT: fmov s16, w21 -; CHECK-GI-NEXT: smov w21, v3.b[8] -; CHECK-GI-NEXT: smov w26, v3.b[5] -; CHECK-GI-NEXT: fmov s17, w23 -; CHECK-GI-NEXT: smov w23, v3.b[0] -; CHECK-GI-NEXT: fmov s18, w25 -; CHECK-GI-NEXT: smov w25, v3.b[3] -; CHECK-GI-NEXT: mov v16.s[1], w22 -; CHECK-GI-NEXT: smov w22, v3.b[1] -; CHECK-GI-NEXT: fmov s19, w6 -; CHECK-GI-NEXT: mov v17.s[1], w24 -; CHECK-GI-NEXT: smov w24, v3.b[2] -; CHECK-GI-NEXT: smov w6, v3.b[7] -; CHECK-GI-NEXT: mul w0, w0, w21 -; CHECK-GI-NEXT: mov v18.s[1], w26 -; CHECK-GI-NEXT: smov w26, v3.b[6] -; CHECK-GI-NEXT: fmov s3, w19 -; CHECK-GI-NEXT: fmov s20, w23 -; CHECK-GI-NEXT: mov v19.s[1], w3 -; CHECK-GI-NEXT: mov v16.s[2], w5 +; CHECK-GI-NEXT: str w2, [sp, #12] // 4-byte Folded Spill +; CHECK-GI-NEXT: mov b5, v2.b[2] +; CHECK-GI-NEXT: mov b6, v2.b[3] +; CHECK-GI-NEXT: mov b7, v2.b[4] +; CHECK-GI-NEXT: mov b16, v2.b[5] +; CHECK-GI-NEXT: mov b17, v2.b[6] +; CHECK-GI-NEXT: mov b18, v2.b[7] +; CHECK-GI-NEXT: mov b19, v2.b[8] +; CHECK-GI-NEXT: mov b20, v2.b[9] +; CHECK-GI-NEXT: mov b21, v2.b[15] +; CHECK-GI-NEXT: mov b3, v2.b[1] +; CHECK-GI-NEXT: fmov w19, s2 +; CHECK-GI-NEXT: mov b22, v1.b[6] +; CHECK-GI-NEXT: fmov w6, s5 +; CHECK-GI-NEXT: mov b5, v2.b[10] +; CHECK-GI-NEXT: fmov w14, s6 +; CHECK-GI-NEXT: mov b6, v2.b[11] +; CHECK-GI-NEXT: fmov w2, s7 +; CHECK-GI-NEXT: stp s17, s18, [sp, #4] // 8-byte Folded Spill +; CHECK-GI-NEXT: mov b7, v2.b[12] +; CHECK-GI-NEXT: fmov w11, s16 +; CHECK-GI-NEXT: sxtb w28, w19 +; CHECK-GI-NEXT: mov b16, v2.b[13] +; CHECK-GI-NEXT: mov b18, v1.b[1] +; CHECK-GI-NEXT: sxtb w6, w6 +; CHECK-GI-NEXT: mov b17, v2.b[14] +; CHECK-GI-NEXT: ldp q4, q2, [x0] +; CHECK-GI-NEXT: fmov w25, s19 +; CHECK-GI-NEXT: fmov w24, s20 +; CHECK-GI-NEXT: fmov w22, s5 +; CHECK-GI-NEXT: mov b5, v1.b[2] +; CHECK-GI-NEXT: fmov w0, s6 +; CHECK-GI-NEXT: sxtb w14, w14 +; CHECK-GI-NEXT: mov b20, v1.b[3] +; CHECK-GI-NEXT: fmov w16, s7 +; CHECK-GI-NEXT: mov b7, v1.b[4] +; CHECK-GI-NEXT: fmov w15, s16 +; CHECK-GI-NEXT: sxtb w25, w25 +; CHECK-GI-NEXT: sxtb w24, w24 +; CHECK-GI-NEXT: mov b16, v1.b[5] +; CHECK-GI-NEXT: fmov w13, s21 +; CHECK-GI-NEXT: sxtb w22, w22 +; CHECK-GI-NEXT: mov b6, v4.b[2] +; CHECK-GI-NEXT: fmov w26, s18 +; CHECK-GI-NEXT: sxtb w0, w0 +; CHECK-GI-NEXT: mov b21, v1.b[7] +; CHECK-GI-NEXT: mov b18, v4.b[4] +; CHECK-GI-NEXT: fmov w7, s3 +; CHECK-GI-NEXT: mov b3, v4.b[1] +; CHECK-GI-NEXT: fmov w12, s17 +; CHECK-GI-NEXT: fmov w5, s5 +; CHECK-GI-NEXT: mov b19, v4.b[3] +; CHECK-GI-NEXT: fmov w4, s20 +; CHECK-GI-NEXT: fmov w3, s7 +; CHECK-GI-NEXT: sxtb w29, w7 +; CHECK-GI-NEXT: mov b17, v4.b[5] +; CHECK-GI-NEXT: fmov w1, s16 +; CHECK-GI-NEXT: sxtb w5, w5 +; CHECK-GI-NEXT: mov b16, v4.b[6] +; CHECK-GI-NEXT: fmov w18, s22 +; CHECK-GI-NEXT: mov b7, v4.b[7] +; CHECK-GI-NEXT: fmov w17, s21 +; CHECK-GI-NEXT: mov b5, v4.b[8] +; CHECK-GI-NEXT: mov b20, v4.b[9] +; CHECK-GI-NEXT: fmov w27, s6 +; CHECK-GI-NEXT: mov b6, v4.b[10] +; CHECK-GI-NEXT: mov b21, v4.b[11] +; CHECK-GI-NEXT: fmov w21, s18 +; CHECK-GI-NEXT: mov b18, v4.b[12] +; CHECK-GI-NEXT: mov b22, v4.b[13] +; CHECK-GI-NEXT: mov b23, v4.b[14] +; CHECK-GI-NEXT: fmov w10, s4 +; CHECK-GI-NEXT: sxtb w27, w27 +; CHECK-GI-NEXT: mov b24, v4.b[15] +; CHECK-GI-NEXT: fmov s4, w25 +; CHECK-GI-NEXT: fmov w30, s3 +; CHECK-GI-NEXT: fmov s3, w28 +; CHECK-GI-NEXT: fmov w9, s5 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: fmov w7, s7 +; CHECK-GI-NEXT: mov b7, v2.b[1] +; CHECK-GI-NEXT: mov v4.h[1], w24 +; CHECK-GI-NEXT: fmov w24, s1 +; CHECK-GI-NEXT: fmov w8, s20 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v3.h[1], w29 +; CHECK-GI-NEXT: fmov w29, s6 +; CHECK-GI-NEXT: fmov s6, w10 +; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: fmov w19, s16 +; CHECK-GI-NEXT: sxtb w24, w24 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov b16, v2.b[3] +; CHECK-GI-NEXT: sxtb w29, w29 +; CHECK-GI-NEXT: fmov w23, s19 +; CHECK-GI-NEXT: mov b19, v2.b[2] +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: fmov s5, w24 +; CHECK-GI-NEXT: sxtb w24, w30 +; CHECK-GI-NEXT: mov v3.h[2], w6 +; CHECK-GI-NEXT: sxtb w6, w26 +; CHECK-GI-NEXT: fmov w28, s21 +; CHECK-GI-NEXT: sxtb w23, w23 +; CHECK-GI-NEXT: mov v6.h[1], w24 +; CHECK-GI-NEXT: fmov w24, s7 +; CHECK-GI-NEXT: fmov s7, w9 +; CHECK-GI-NEXT: fmov w9, s19 +; CHECK-GI-NEXT: mov v5.h[1], w6 +; CHECK-GI-NEXT: mov v4.h[2], w22 +; CHECK-GI-NEXT: fmov w20, s17 +; CHECK-GI-NEXT: mov b17, v2.b[4] +; CHECK-GI-NEXT: sxtb w24, w24 +; CHECK-GI-NEXT: mov v3.h[3], w14 +; CHECK-GI-NEXT: sxtb w14, w2 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v7.h[1], w8 +; CHECK-GI-NEXT: fmov w8, s16 +; CHECK-GI-NEXT: fmov s16, w10 +; CHECK-GI-NEXT: mov v6.h[2], w27 +; CHECK-GI-NEXT: mov v5.h[2], w5 +; CHECK-GI-NEXT: fmov w25, s18 +; CHECK-GI-NEXT: mov v4.h[3], w0 +; CHECK-GI-NEXT: sxtb w0, w4 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov b18, v2.b[5] +; CHECK-GI-NEXT: fmov w10, s17 +; CHECK-GI-NEXT: mov v16.h[1], w24 +; CHECK-GI-NEXT: mov v7.h[2], w29 +; CHECK-GI-NEXT: mov v3.h[4], w14 +; CHECK-GI-NEXT: sxtb w14, w25 +; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v6.h[3], w23 +; CHECK-GI-NEXT: mov v5.h[3], w0 +; CHECK-GI-NEXT: fmov w26, s22 +; CHECK-GI-NEXT: mov b19, v2.b[6] +; CHECK-GI-NEXT: fmov w27, s18 +; CHECK-GI-NEXT: mov v16.h[2], w9 +; CHECK-GI-NEXT: sxtb w9, w28 +; CHECK-GI-NEXT: fmov w22, s23 +; CHECK-GI-NEXT: mov b17, v2.b[7] +; CHECK-GI-NEXT: fmov w6, s24 ; CHECK-GI-NEXT: mov v0.s[1], wzr -; CHECK-GI-NEXT: mov v6.s[2], w10 -; CHECK-GI-NEXT: fmov s21, w0 -; CHECK-GI-NEXT: mov v17.s[2], w20 -; CHECK-GI-NEXT: mov v4.s[3], w11 -; CHECK-GI-NEXT: ldp x20, x19, [sp, #48] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v3.s[1], w7 -; CHECK-GI-NEXT: mov v20.s[1], w22 -; CHECK-GI-NEXT: ldp x22, x21, [sp, #32] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v18.s[2], w26 -; CHECK-GI-NEXT: mov v21.s[1], wzr -; CHECK-GI-NEXT: mov v16.s[3], w18 -; CHECK-GI-NEXT: mov v17.s[3], w4 -; CHECK-GI-NEXT: mov v7.s[2], w13 -; CHECK-GI-NEXT: mov v5.s[3], w14 -; CHECK-GI-NEXT: mov v19.s[2], w16 -; CHECK-GI-NEXT: mov v3.s[2], w1 +; CHECK-GI-NEXT: mov v7.h[3], w9 +; CHECK-GI-NEXT: sxtb w9, w11 +; CHECK-GI-NEXT: sxtb w11, w21 +; CHECK-GI-NEXT: fmov w24, s19 +; CHECK-GI-NEXT: mov v16.h[3], w8 +; CHECK-GI-NEXT: sxtb w8, w16 +; CHECK-GI-NEXT: sxtb w16, w3 +; CHECK-GI-NEXT: mov v6.h[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #4] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v3.h[5], w9 +; CHECK-GI-NEXT: sxtb w9, w15 +; CHECK-GI-NEXT: sxtb w15, w27 +; CHECK-GI-NEXT: mov v7.h[4], w14 +; CHECK-GI-NEXT: sxtb w14, w1 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: mov v4.h[4], w8 +; CHECK-GI-NEXT: sxtb w8, w20 +; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v5.h[4], w16 +; CHECK-GI-NEXT: mov v16.h[4], w10 +; CHECK-GI-NEXT: sxtb w10, w26 +; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v6.h[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #8] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v7.h[5], w10 +; CHECK-GI-NEXT: sxtb w10, w12 +; CHECK-GI-NEXT: sxtb w12, w18 +; CHECK-GI-NEXT: mov v4.h[5], w9 +; CHECK-GI-NEXT: sxtb w9, w19 +; CHECK-GI-NEXT: mov v5.h[5], w14 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov v16.h[5], w15 +; CHECK-GI-NEXT: mov v3.h[6], w11 +; CHECK-GI-NEXT: sxtb w11, w22 +; CHECK-GI-NEXT: mov v6.h[6], w9 +; CHECK-GI-NEXT: sxtb w9, w13 +; CHECK-GI-NEXT: sxtb w13, w24 ; CHECK-GI-NEXT: mov v0.s[2], wzr -; CHECK-GI-NEXT: mov v20.s[2], w24 -; CHECK-GI-NEXT: ldp x24, x23, [sp, #16] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v18.s[3], w6 -; CHECK-GI-NEXT: mov v21.s[2], wzr -; CHECK-GI-NEXT: mul v2.4s, v2.4s, v16.4s -; CHECK-GI-NEXT: mul v4.4s, v4.4s, v17.4s -; CHECK-GI-NEXT: mov v1.s[3], w8 -; CHECK-GI-NEXT: mov v6.s[3], w9 -; CHECK-GI-NEXT: mov v7.s[3], w12 -; CHECK-GI-NEXT: mov v19.s[3], w15 -; CHECK-GI-NEXT: mov v3.s[3], w17 -; CHECK-GI-NEXT: mov v20.s[3], w25 +; CHECK-GI-NEXT: mov v7.h[6], w11 +; CHECK-GI-NEXT: fmov w11, s17 +; CHECK-GI-NEXT: mov v4.h[6], w10 +; CHECK-GI-NEXT: sxtb w10, w7 +; CHECK-GI-NEXT: mov v5.h[6], w12 +; CHECK-GI-NEXT: mov v16.h[6], w13 +; CHECK-GI-NEXT: mov v3.h[7], w8 +; CHECK-GI-NEXT: sxtb w8, w6 +; CHECK-GI-NEXT: smov w12, v1.b[8] +; CHECK-GI-NEXT: mov v6.h[7], w10 +; CHECK-GI-NEXT: sxtb w10, w17 +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: mov v4.h[7], w9 +; CHECK-GI-NEXT: mov v7.h[7], w8 +; CHECK-GI-NEXT: smov w8, v2.b[8] +; CHECK-GI-NEXT: mov v5.h[7], w10 +; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v16.h[7], w11 ; CHECK-GI-NEXT: mov v0.s[3], wzr -; CHECK-GI-NEXT: mul v5.4s, v5.4s, v18.4s -; CHECK-GI-NEXT: mov v21.s[3], wzr -; CHECK-GI-NEXT: mla v2.4s, v1.4s, v19.4s -; CHECK-GI-NEXT: mla v4.4s, v6.4s, v3.4s -; CHECK-GI-NEXT: mla v5.4s, v7.4s, v20.4s -; CHECK-GI-NEXT: add v0.4s, v21.4s, v0.4s -; CHECK-GI-NEXT: add v1.4s, v2.4s, v4.4s -; CHECK-GI-NEXT: add v0.4s, v5.4s, v0.4s +; CHECK-GI-NEXT: mul v3.8h, v3.8h, v6.8h +; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload +; CHECK-GI-NEXT: mul v2.8h, v4.8h, v7.8h +; CHECK-GI-NEXT: mul w16, w12, w8 +; CHECK-GI-NEXT: mul v1.8h, v5.8h, v16.8h +; CHECK-GI-NEXT: smov w17, v3.h[0] +; CHECK-GI-NEXT: smov w0, v3.h[4] +; CHECK-GI-NEXT: sxth w16, w16 +; CHECK-GI-NEXT: smov w2, v2.h[0] +; CHECK-GI-NEXT: smov w4, v2.h[4] +; CHECK-GI-NEXT: smov w18, v3.h[1] +; CHECK-GI-NEXT: smov w1, v3.h[5] +; CHECK-GI-NEXT: smov w3, v2.h[1] +; CHECK-GI-NEXT: smov w5, v2.h[5] +; CHECK-GI-NEXT: smov w6, v1.h[0] +; CHECK-GI-NEXT: smov w19, v1.h[4] +; CHECK-GI-NEXT: smov w7, v1.h[1] +; CHECK-GI-NEXT: smov w20, v1.h[5] +; CHECK-GI-NEXT: smov w10, v3.h[2] +; CHECK-GI-NEXT: smov w8, v3.h[3] +; CHECK-GI-NEXT: smov w11, v3.h[6] +; CHECK-GI-NEXT: smov w9, v3.h[7] +; CHECK-GI-NEXT: fmov s3, w17 +; CHECK-GI-NEXT: fmov s4, w0 +; CHECK-GI-NEXT: fmov s5, w2 +; CHECK-GI-NEXT: fmov s6, w4 +; CHECK-GI-NEXT: fmov s7, w6 +; CHECK-GI-NEXT: fmov s16, w19 +; CHECK-GI-NEXT: fmov s17, w16 +; CHECK-GI-NEXT: smov w12, v2.h[2] +; CHECK-GI-NEXT: smov w13, v2.h[6] +; CHECK-GI-NEXT: smov w14, v1.h[2] +; CHECK-GI-NEXT: smov w15, v1.h[6] +; CHECK-GI-NEXT: mov v3.s[1], w18 +; CHECK-GI-NEXT: mov v4.s[1], w1 +; CHECK-GI-NEXT: mov v5.s[1], w3 +; CHECK-GI-NEXT: mov v6.s[1], w5 +; CHECK-GI-NEXT: mov v7.s[1], w7 +; CHECK-GI-NEXT: mov v16.s[1], w20 +; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v17.s[1], wzr +; CHECK-GI-NEXT: smov w16, v2.h[3] +; CHECK-GI-NEXT: smov w17, v2.h[7] +; CHECK-GI-NEXT: smov w18, v1.h[3] +; CHECK-GI-NEXT: smov w0, v1.h[7] +; CHECK-GI-NEXT: mov v3.s[2], w10 +; CHECK-GI-NEXT: mov v4.s[2], w11 +; CHECK-GI-NEXT: mov v5.s[2], w12 +; CHECK-GI-NEXT: mov v6.s[2], w13 +; CHECK-GI-NEXT: mov v7.s[2], w14 +; CHECK-GI-NEXT: mov v16.s[2], w15 +; CHECK-GI-NEXT: mov v17.s[2], wzr +; CHECK-GI-NEXT: mov v3.s[3], w8 +; CHECK-GI-NEXT: mov v4.s[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #12] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v5.s[3], w16 +; CHECK-GI-NEXT: mov v6.s[3], w17 +; CHECK-GI-NEXT: mov v7.s[3], w18 +; CHECK-GI-NEXT: mov v16.s[3], w0 +; CHECK-GI-NEXT: mov v17.s[3], wzr +; CHECK-GI-NEXT: add v1.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: add v2.4s, v5.4s, v6.4s +; CHECK-GI-NEXT: add v3.4s, v7.4s, v16.4s +; CHECK-GI-NEXT: add v0.4s, v17.4s, v0.4s +; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s ; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ldp x26, x25, [sp], #64 // 16-byte Folded Reload +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: add sp, sp, #112 ; CHECK-GI-NEXT: ret entry: %0 = load <25 x i8>, ptr %a @@ -2948,349 +3331,535 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 ; ; CHECK-GI-LABEL: test_sdot_v25i8_double: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: stp d11, d10, [sp, #-48]! // 16-byte Folded Spill -; CHECK-GI-NEXT: stp d9, d8, [sp, #16] // 16-byte Folded Spill -; CHECK-GI-NEXT: str x29, [sp, #32] // 8-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 48 +; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 ; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: .cfi_offset b8, -24 -; CHECK-GI-NEXT: .cfi_offset b9, -32 -; CHECK-GI-NEXT: .cfi_offset b10, -40 -; CHECK-GI-NEXT: .cfi_offset b11, -48 -; CHECK-GI-NEXT: sxtb w8, w0 -; CHECK-GI-NEXT: sxtb w10, w4 -; CHECK-GI-NEXT: sxtb w9, w1 -; CHECK-GI-NEXT: sxtb w11, w2 -; CHECK-GI-NEXT: sxtb w13, w6 -; CHECK-GI-NEXT: ldr w12, [sp, #72] +; CHECK-GI-NEXT: lsl w8, w0, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #16] +; CHECK-GI-NEXT: lsl w10, w1, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #24] +; CHECK-GI-NEXT: lsl w12, w4, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #56] +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #64] ; CHECK-GI-NEXT: fmov s2, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #48] -; CHECK-GI-NEXT: fmov s4, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #80] -; CHECK-GI-NEXT: ldr w14, [sp, #128] -; CHECK-GI-NEXT: ldr w15, [sp, #152] -; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: lsl w8, w11, #8 +; CHECK-GI-NEXT: lsl w11, w2, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: fmov s4, w9 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #112] +; CHECK-GI-NEXT: mov v2.h[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #32] +; CHECK-GI-NEXT: sbfx w9, w11, #8, #8 +; CHECK-GI-NEXT: lsl w11, w3, #8 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 ; CHECK-GI-NEXT: fmov s1, wzr +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v4.h[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #152] +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 ; CHECK-GI-NEXT: fmov s0, wzr -; CHECK-GI-NEXT: mov v2.s[1], w9 -; CHECK-GI-NEXT: sxtb w9, w5 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: fmov s3, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #88] -; CHECK-GI-NEXT: ldr x29, [sp, #32] // 8-byte Folded Reload -; CHECK-GI-NEXT: mov v4.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #56] -; CHECK-GI-NEXT: fmov s5, w10 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w10, w3 +; CHECK-GI-NEXT: mov v2.h[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #40] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: lsl w8, w8, #8 ; CHECK-GI-NEXT: mov v1.s[1], wzr -; CHECK-GI-NEXT: mov v2.s[2], w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w11, [sp, #64] -; CHECK-GI-NEXT: mov v5.s[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #104] +; CHECK-GI-NEXT: mov v4.h[2], w10 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #160] +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 ; CHECK-GI-NEXT: mov v0.s[1], wzr -; CHECK-GI-NEXT: mov v3.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #96] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v4.s[2], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #120] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v2.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #112] -; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v2.h[3], w11 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #48] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: fmov s3, w8 ; CHECK-GI-NEXT: mov v1.s[2], wzr +; CHECK-GI-NEXT: mov v4.h[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #80] +; CHECK-GI-NEXT: lsl w8, w11, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #168] ; CHECK-GI-NEXT: mov v0.s[2], wzr -; CHECK-GI-NEXT: mov v3.s[2], w11 -; CHECK-GI-NEXT: sxtb w11, w10 -; CHECK-GI-NEXT: mov v5.s[2], w9 -; CHECK-GI-NEXT: sxtb w9, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #144] -; CHECK-GI-NEXT: ldr w10, [sp, #136] -; CHECK-GI-NEXT: fmov s6, w11 -; CHECK-GI-NEXT: sxtb w11, w7 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v2.h[4], w12 +; CHECK-GI-NEXT: lsl w12, w5, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: mov v3.h[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #88] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v4.h[4], w8 +; CHECK-GI-NEXT: lsl w8, w10, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #176] +; CHECK-GI-NEXT: mov v2.h[5], w12 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: lsl w12, w6, #8 +; CHECK-GI-NEXT: fmov s6, w9 +; CHECK-GI-NEXT: sbfx w15, w8, #8, #8 +; CHECK-GI-NEXT: lsl w9, w10, #8 +; CHECK-GI-NEXT: mov v3.h[2], w11 +; CHECK-GI-NEXT: sbfx w11, w12, #8, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #96] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v4.h[5], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #224] +; CHECK-GI-NEXT: mov v6.h[1], w15 +; CHECK-GI-NEXT: mov v2.h[6], w11 +; CHECK-GI-NEXT: lsl w15, w7, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #184] +; CHECK-GI-NEXT: ldr w12, [sp, #104] +; CHECK-GI-NEXT: mov v3.h[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #216] +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v2.h[7], w15 +; CHECK-GI-NEXT: lsl w15, w9, #8 +; CHECK-GI-NEXT: mov v4.h[6], w14 +; CHECK-GI-NEXT: mov v6.h[2], w10 +; CHECK-GI-NEXT: lsl w10, w13, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: sbfx w13, w15, #8, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #288] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: mov v3.h[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #192] +; CHECK-GI-NEXT: fmov s5, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #232] +; CHECK-GI-NEXT: ldr w9, [sp, #120] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v6.h[3], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #72] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 ; CHECK-GI-NEXT: mov v1.s[3], wzr -; CHECK-GI-NEXT: mov v5.s[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #184] -; CHECK-GI-NEXT: mov v4.s[3], w11 -; CHECK-GI-NEXT: mov v6.s[1], w9 -; CHECK-GI-NEXT: fmov s7, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #216] -; CHECK-GI-NEXT: sxtb w9, w12 -; CHECK-GI-NEXT: sxtb w12, w14 -; CHECK-GI-NEXT: sxtb w14, w15 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: ldr w11, [sp, #160] -; CHECK-GI-NEXT: mov v7.s[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #224] -; CHECK-GI-NEXT: mov v3.s[3], w9 -; CHECK-GI-NEXT: mov v6.s[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #192] -; CHECK-GI-NEXT: fmov s16, w8 -; CHECK-GI-NEXT: fmov s18, w13 -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w9, [sp, #168] -; CHECK-GI-NEXT: ldr w13, [sp, #208] -; CHECK-GI-NEXT: mov v7.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #256] -; CHECK-GI-NEXT: ldr w8, [sp, #176] -; CHECK-GI-NEXT: mov v16.s[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #200] -; CHECK-GI-NEXT: mov v18.s[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #232] -; CHECK-GI-NEXT: mov v6.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #248] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: mov v16.s[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #240] -; CHECK-GI-NEXT: mov v7.s[3], w9 -; CHECK-GI-NEXT: mov v18.s[2], w14 -; CHECK-GI-NEXT: fmov s17, w10 +; CHECK-GI-NEXT: mov v5.h[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #280] +; CHECK-GI-NEXT: sbfx w15, w11, #8, #8 +; CHECK-GI-NEXT: sbfx w12, w13, #8, #8 +; CHECK-GI-NEXT: lsl w13, w14, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #240] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v3.h[5], w15 +; CHECK-GI-NEXT: lsl w15, w16, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: mov v5.h[2], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #296] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: fmov s7, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #200] +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v6.h[4], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #304] +; CHECK-GI-NEXT: ldr w11, [sp, #128] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v5.h[3], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #208] +; CHECK-GI-NEXT: mov v7.h[1], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #248] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v6.h[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #272] +; CHECK-GI-NEXT: mov v3.h[6], w10 +; CHECK-GI-NEXT: lsl w10, w14, #8 +; CHECK-GI-NEXT: sbfx w14, w15, #8, #8 +; CHECK-GI-NEXT: mov v7.h[2], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #256] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: ldr w15, [sp, #320] +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: mov v5.h[4], w13 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #312] +; CHECK-GI-NEXT: mov v3.h[7], w10 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v4.h[7], w8 +; CHECK-GI-NEXT: mov v7.h[3], w14 ; CHECK-GI-NEXT: ldr w14, [sp, #264] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w9, [sp, #288] -; CHECK-GI-NEXT: ldr w10, [sp, #272] -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: ldr w15, [sp, #392] -; CHECK-GI-NEXT: mov v17.s[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #280] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v18.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #312] -; CHECK-GI-NEXT: mov v16.s[3], w13 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: ldr w13, [sp, #296] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w8, w9, #8, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #136] +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: mov v5.h[5], w12 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mul v16.8h, v2.8h, v3.8h +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: sbfx w12, w14, #8, #8 +; CHECK-GI-NEXT: lsl w14, w15, #8 +; CHECK-GI-NEXT: mov v6.h[6], w11 +; CHECK-GI-NEXT: mov v7.h[4], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #328] +; CHECK-GI-NEXT: ldr w10, [sp, #144] +; CHECK-GI-NEXT: mov v5.h[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #336] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: smov w9, v16.h[0] +; CHECK-GI-NEXT: smov w15, v16.h[4] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: smov w17, v16.h[5] ; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w12, w12 ; CHECK-GI-NEXT: mov v0.s[3], wzr -; CHECK-GI-NEXT: mov v17.s[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #320] -; CHECK-GI-NEXT: fmov s20, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #344] -; CHECK-GI-NEXT: fmov s19, w12 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: ldr w12, [sp, #304] -; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v20.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #352] -; CHECK-GI-NEXT: mov v19.s[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #328] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: fmov s21, w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w11, [sp, #336] +; CHECK-GI-NEXT: sbfx w11, w13, #8, #8 +; CHECK-GI-NEXT: smov w13, v16.h[1] +; CHECK-GI-NEXT: mov v7.h[5], w14 +; CHECK-GI-NEXT: mov v5.h[7], w8 +; CHECK-GI-NEXT: ldr w14, [sp, #344] +; CHECK-GI-NEXT: ldr w8, [sp, #352] +; CHECK-GI-NEXT: fmov s2, w9 +; CHECK-GI-NEXT: fmov s3, w15 +; CHECK-GI-NEXT: lsl w9, w12, #8 +; CHECK-GI-NEXT: sbfx w12, w16, #8, #8 ; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: mov v17.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #376] -; CHECK-GI-NEXT: mov v20.s[2], w13 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: ldr w13, [sp, #368] -; CHECK-GI-NEXT: mov v21.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #360] -; CHECK-GI-NEXT: mov v19.s[2], w14 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: ldr w14, [sp, #384] -; CHECK-GI-NEXT: mla v4.4s, v2.4s, v16.4s -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v20.s[3], w12 -; CHECK-GI-NEXT: sxtb w12, w13 -; CHECK-GI-NEXT: mul w10, w8, w10 -; CHECK-GI-NEXT: mov v21.s[2], w9 -; CHECK-GI-NEXT: mov v19.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #416] -; CHECK-GI-NEXT: sxtb w13, w14 -; CHECK-GI-NEXT: sxtb w14, w15 -; CHECK-GI-NEXT: ldr w9, [sp, #400] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: fmov s22, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #432] -; CHECK-GI-NEXT: fmov s23, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #448] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v21.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #424] -; CHECK-GI-NEXT: fmov s25, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #480] -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v23.s[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #456] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: fmov s24, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #440] -; CHECK-GI-NEXT: mov v25.s[1], w12 +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: mov v7.h[6], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #360] +; CHECK-GI-NEXT: smov w15, v16.h[3] +; CHECK-GI-NEXT: mov v2.s[1], w13 +; CHECK-GI-NEXT: smov w13, v16.h[2] +; CHECK-GI-NEXT: mov v6.h[7], w12 +; CHECK-GI-NEXT: smov w12, v16.h[6] +; CHECK-GI-NEXT: mov v3.s[1], w17 +; CHECK-GI-NEXT: mul v18.8h, v4.8h, v5.8h +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: sbfx w16, w9, #8, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #368] +; CHECK-GI-NEXT: mov v2.s[2], w13 +; CHECK-GI-NEXT: smov w13, v16.h[7] +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v3.s[2], w12 +; CHECK-GI-NEXT: sbfx w12, w8, #8, #8 +; CHECK-GI-NEXT: mul w8, w10, w14 +; CHECK-GI-NEXT: smov w10, v18.h[0] +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #376] +; CHECK-GI-NEXT: fmov s16, w12 +; CHECK-GI-NEXT: smov w12, v18.h[1] +; CHECK-GI-NEXT: mov v7.h[7], w16 +; CHECK-GI-NEXT: mov v2.s[3], w15 +; CHECK-GI-NEXT: smov w15, v18.h[4] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v3.s[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #416] +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: fmov s4, w10 +; CHECK-GI-NEXT: mov v16.h[1], w11 +; CHECK-GI-NEXT: ldr w10, [sp, #424] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #384] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: fmov s5, w15 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: ldr w15, [sp, #432] +; CHECK-GI-NEXT: mov v4.s[1], w12 +; CHECK-GI-NEXT: smov w12, v18.h[5] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v16.h[2], w9 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: fmov s17, w13 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mul v7.8h, v6.8h, v7.8h +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #392] +; CHECK-GI-NEXT: ldr w13, [sp, #400] +; CHECK-GI-NEXT: mov v5.s[1], w12 +; CHECK-GI-NEXT: smov w12, v18.h[2] +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v17.h[1], w10 +; CHECK-GI-NEXT: mov v16.h[3], w14 +; CHECK-GI-NEXT: ldr w10, [sp, #440] +; CHECK-GI-NEXT: smov w14, v18.h[6] +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #456] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sxth w8, w8 +; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: mov v4.s[2], w12 +; CHECK-GI-NEXT: smov w12, v18.h[3] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v17.h[2], w15 +; CHECK-GI-NEXT: mov v16.h[4], w11 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: mov v5.s[2], w14 +; CHECK-GI-NEXT: smov w14, v18.h[7] +; CHECK-GI-NEXT: ldr w15, [sp, #448] +; CHECK-GI-NEXT: ldr w11, [sp, #408] +; CHECK-GI-NEXT: mov v4.s[3], w12 +; CHECK-GI-NEXT: smov w12, v7.h[0] +; CHECK-GI-NEXT: mov v17.h[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #480] +; CHECK-GI-NEXT: mov v16.h[5], w9 +; CHECK-GI-NEXT: lsl w9, w13, #8 +; CHECK-GI-NEXT: lsl w13, w15, #8 +; CHECK-GI-NEXT: mov v5.s[3], w14 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: smov w14, v7.h[1] +; CHECK-GI-NEXT: lsl w15, w16, #8 +; CHECK-GI-NEXT: fmov s6, w12 ; CHECK-GI-NEXT: ldr w12, [sp, #488] -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: fmov s26, w11 -; CHECK-GI-NEXT: ldr w15, [sp, #504] -; CHECK-GI-NEXT: ldr w11, [sp, #472] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v24.s[1], w14 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v17.h[4], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #496] +; CHECK-GI-NEXT: fmov s18, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #552] +; CHECK-GI-NEXT: mov v6.s[1], w14 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 ; CHECK-GI-NEXT: ldr w14, [sp, #464] -; CHECK-GI-NEXT: mov v23.s[2], w9 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: ldr w8, [sp, #408] -; CHECK-GI-NEXT: mov v26.s[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #496] -; CHECK-GI-NEXT: mov v25.s[2], w10 +; CHECK-GI-NEXT: mov v16.h[6], w9 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v18.h[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #560] +; CHECK-GI-NEXT: mov v17.h[5], w15 +; CHECK-GI-NEXT: sbfx w15, w10, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 ; CHECK-GI-NEXT: ldr w10, [sp, #512] -; CHECK-GI-NEXT: sxtb w9, w14 -; CHECK-GI-NEXT: ldr w14, [sp, #520] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v22.s[1], wzr -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v24.s[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #528] -; CHECK-GI-NEXT: mov v26.s[2], w12 -; CHECK-GI-NEXT: sxtb w12, w13 -; CHECK-GI-NEXT: sxtb w13, w15 -; CHECK-GI-NEXT: fmov s27, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #584] -; CHECK-GI-NEXT: ldr w15, [sp, #552] -; CHECK-GI-NEXT: mov v25.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #544] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v24.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #560] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v26.s[3], w13 -; CHECK-GI-NEXT: sxtb w13, w14 -; CHECK-GI-NEXT: sxtb w14, w15 -; CHECK-GI-NEXT: fmov s29, w10 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: fmov s28, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #616] -; CHECK-GI-NEXT: mov v27.s[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #592] -; CHECK-GI-NEXT: ldr w15, [sp, #568] -; CHECK-GI-NEXT: mov v23.s[3], w8 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w8, [sp, #536] -; CHECK-GI-NEXT: ldr w10, [sp, #576] -; CHECK-GI-NEXT: mov v28.s[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #624] -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: fmov s30, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #600] -; CHECK-GI-NEXT: mov v27.s[2], w9 -; CHECK-GI-NEXT: mov v29.s[1], w13 -; CHECK-GI-NEXT: sxtb w13, w14 -; CHECK-GI-NEXT: sxtb w14, w15 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w9, [sp, #608] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v30.s[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #632] -; CHECK-GI-NEXT: mov v28.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #640] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: mov v29.s[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #648] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v27.s[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #664] -; CHECK-GI-NEXT: mov v30.s[2], w13 -; CHECK-GI-NEXT: mov v28.s[3], w14 +; CHECK-GI-NEXT: fmov s19, w15 +; CHECK-GI-NEXT: ldr w15, [sp, #616] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v16.h[7], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #504] +; CHECK-GI-NEXT: mov v18.h[2], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #568] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: mov v17.h[6], w14 +; CHECK-GI-NEXT: lsl w14, w15, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: ldr w15, [sp, #576] +; CHECK-GI-NEXT: mov v19.h[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #624] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: sbfx w16, w11, #8, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: fmov s20, w14 ; CHECK-GI-NEXT: ldr w14, [sp, #680] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w13, [sp, #656] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: mov v29.s[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #688] -; CHECK-GI-NEXT: fmov s31, w12 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: ldr w12, [sp, #752] -; CHECK-GI-NEXT: mov v30.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #744] -; CHECK-GI-NEXT: fmov s8, w14 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w14, [sp, #712] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v31.s[1], w13 +; CHECK-GI-NEXT: mov v18.h[3], w16 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: mov v19.h[2], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #632] +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: mov v20.h[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #688] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v18.h[4], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #584] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v19.h[3], w15 +; CHECK-GI-NEXT: fmov s21, w14 +; CHECK-GI-NEXT: ldr w15, [sp, #640] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v20.h[2], w13 ; CHECK-GI-NEXT: ldr w13, [sp, #696] -; CHECK-GI-NEXT: mov v8.s[1], w9 -; CHECK-GI-NEXT: sxtb w14, w14 -; CHECK-GI-NEXT: ldr w9, [sp, #720] -; CHECK-GI-NEXT: fmov s9, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #776] -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: fmov s10, w14 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v22.s[2], wzr -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v31.s[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #704] -; CHECK-GI-NEXT: mov v9.s[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #760] -; CHECK-GI-NEXT: mov v8.s[2], w13 -; CHECK-GI-NEXT: mul w10, w10, w11 -; CHECK-GI-NEXT: mov v10.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #728] -; CHECK-GI-NEXT: sxtb w11, w12 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mul v5.4s, v5.4s, v20.4s -; CHECK-GI-NEXT: mul v7.4s, v7.4s, v21.4s -; CHECK-GI-NEXT: mul v18.4s, v25.4s, v30.4s -; CHECK-GI-NEXT: mov v22.s[3], wzr -; CHECK-GI-NEXT: fmov s11, w10 -; CHECK-GI-NEXT: mov v9.s[2], w11 -; CHECK-GI-NEXT: ldr w10, [sp, #768] -; CHECK-GI-NEXT: mov v8.s[3], w8 -; CHECK-GI-NEXT: sxtb w8, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #672] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #520] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v21.h[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #592] +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: mov v19.h[4], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #704] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v20.h[3], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #648] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v21.h[2], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #600] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: mov v18.h[5], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #712] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v19.h[5], w12 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: ldr w12, [sp, #656] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v21.h[3], w10 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #608] +; CHECK-GI-NEXT: mov v20.h[4], w15 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #528] +; CHECK-GI-NEXT: ldr w15, [sp, #664] +; CHECK-GI-NEXT: mov v19.h[6], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #720] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: mov v21.h[4], w11 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w16, w10, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: mov v20.h[5], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #728] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v19.h[7], w16 +; CHECK-GI-NEXT: ldr w9, [sp, #472] +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v18.h[6], w14 +; CHECK-GI-NEXT: sbfx w14, w15, #8, #8 +; CHECK-GI-NEXT: mov v21.h[5], w13 +; CHECK-GI-NEXT: ldr w15, [sp, #672] +; CHECK-GI-NEXT: ldr w11, [sp, #536] +; CHECK-GI-NEXT: ldr w13, [sp, #736] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: mov v20.h[6], w14 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mul v19.8h, v16.8h, v19.8h +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v21.h[6], w12 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: smov w14, v7.h[2] +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: smov w12, v7.h[4] +; CHECK-GI-NEXT: mov v17.h[7], w9 +; CHECK-GI-NEXT: mov v20.h[7], w15 +; CHECK-GI-NEXT: smov w9, v7.h[5] +; CHECK-GI-NEXT: mov v18.h[7], w11 +; CHECK-GI-NEXT: smov w11, v19.h[4] +; CHECK-GI-NEXT: ldr w15, [sp, #744] +; CHECK-GI-NEXT: mov v21.h[7], w13 +; CHECK-GI-NEXT: mov v6.s[2], w14 +; CHECK-GI-NEXT: smov w14, v19.h[0] +; CHECK-GI-NEXT: fmov s16, w12 +; CHECK-GI-NEXT: smov w13, v19.h[5] +; CHECK-GI-NEXT: smov w12, v19.h[1] +; CHECK-GI-NEXT: mul v20.8h, v17.8h, v20.8h +; CHECK-GI-NEXT: ldr w10, [sp, #544] +; CHECK-GI-NEXT: add v3.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: mul v22.8h, v18.8h, v21.8h +; CHECK-GI-NEXT: fmov s18, w11 +; CHECK-GI-NEXT: mov v16.s[1], w9 +; CHECK-GI-NEXT: fmov s17, w14 +; CHECK-GI-NEXT: smov w14, v7.h[6] +; CHECK-GI-NEXT: smov w11, v19.h[2] +; CHECK-GI-NEXT: smov w9, v7.h[3] ; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mla v5.4s, v3.4s, v17.4s -; CHECK-GI-NEXT: mov v11.s[1], wzr -; CHECK-GI-NEXT: mov v10.s[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #736] -; CHECK-GI-NEXT: mov v9.s[3], w10 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mla v7.4s, v6.4s, v19.4s -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mul v20.4s, v26.4s, v8.4s -; CHECK-GI-NEXT: mla v18.4s, v23.4s, v29.4s -; CHECK-GI-NEXT: mov v31.s[3], w9 -; CHECK-GI-NEXT: add v1.4s, v22.4s, v1.4s -; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s -; CHECK-GI-NEXT: mov v11.s[2], wzr -; CHECK-GI-NEXT: mov v10.s[3], w8 -; CHECK-GI-NEXT: mul v21.4s, v28.4s, v9.4s -; CHECK-GI-NEXT: ldp d9, d8, [sp, #16] // 16-byte Folded Reload -; CHECK-GI-NEXT: add v1.4s, v7.4s, v1.4s -; CHECK-GI-NEXT: mla v20.4s, v24.4s, v31.4s -; CHECK-GI-NEXT: mov v11.s[3], wzr -; CHECK-GI-NEXT: mla v21.4s, v27.4s, v10.4s +; CHECK-GI-NEXT: fmov s21, w8 +; CHECK-GI-NEXT: mov v18.s[1], w13 +; CHECK-GI-NEXT: sxtb w13, w15 +; CHECK-GI-NEXT: smov w15, v20.h[0] +; CHECK-GI-NEXT: mov v17.s[1], w12 +; CHECK-GI-NEXT: smov w8, v7.h[7] +; CHECK-GI-NEXT: smov w12, v19.h[6] +; CHECK-GI-NEXT: mov v16.s[2], w14 +; CHECK-GI-NEXT: smov w14, v20.h[1] +; CHECK-GI-NEXT: mul w10, w10, w13 +; CHECK-GI-NEXT: smov w13, v20.h[4] +; CHECK-GI-NEXT: smov w16, v20.h[5] +; CHECK-GI-NEXT: mov v21.s[1], wzr +; CHECK-GI-NEXT: fmov s7, w15 +; CHECK-GI-NEXT: smov w15, v20.h[2] +; CHECK-GI-NEXT: mov v6.s[3], w9 +; CHECK-GI-NEXT: mov v17.s[2], w11 +; CHECK-GI-NEXT: smov w11, v22.h[0] +; CHECK-GI-NEXT: sxth w10, w10 +; CHECK-GI-NEXT: mov v18.s[2], w12 +; CHECK-GI-NEXT: smov w12, v22.h[1] +; CHECK-GI-NEXT: mov v16.s[3], w8 +; CHECK-GI-NEXT: mov v7.s[1], w14 +; CHECK-GI-NEXT: smov w14, v22.h[4] +; CHECK-GI-NEXT: fmov s23, w13 +; CHECK-GI-NEXT: smov w13, v22.h[5] +; CHECK-GI-NEXT: fmov s26, w10 +; CHECK-GI-NEXT: smov w10, v19.h[7] +; CHECK-GI-NEXT: fmov s24, w11 +; CHECK-GI-NEXT: smov w11, v20.h[6] +; CHECK-GI-NEXT: mov v21.s[2], wzr +; CHECK-GI-NEXT: mov v23.s[1], w16 +; CHECK-GI-NEXT: add v4.4s, v6.4s, v16.4s +; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: fmov s25, w14 +; CHECK-GI-NEXT: smov w14, v22.h[2] +; CHECK-GI-NEXT: mov v26.s[1], wzr +; CHECK-GI-NEXT: mov v24.s[1], w12 +; CHECK-GI-NEXT: smov w12, v19.h[3] +; CHECK-GI-NEXT: mov v7.s[2], w15 +; CHECK-GI-NEXT: smov w15, v20.h[3] +; CHECK-GI-NEXT: mov v18.s[3], w10 +; CHECK-GI-NEXT: mov v21.s[3], wzr +; CHECK-GI-NEXT: mov v25.s[1], w13 +; CHECK-GI-NEXT: smov w13, v22.h[6] +; CHECK-GI-NEXT: mov v23.s[2], w11 +; CHECK-GI-NEXT: smov w11, v20.h[7] +; CHECK-GI-NEXT: mov v26.s[2], wzr +; CHECK-GI-NEXT: mov v24.s[2], w14 +; CHECK-GI-NEXT: smov w14, v22.h[3] +; CHECK-GI-NEXT: mov v17.s[3], w12 +; CHECK-GI-NEXT: mov v7.s[3], w15 +; CHECK-GI-NEXT: add v1.4s, v21.4s, v1.4s +; CHECK-GI-NEXT: mov v25.s[2], w13 +; CHECK-GI-NEXT: smov w13, v22.h[7] +; CHECK-GI-NEXT: mov v23.s[3], w11 +; CHECK-GI-NEXT: mov v26.s[3], wzr +; CHECK-GI-NEXT: mov v24.s[3], w14 +; CHECK-GI-NEXT: add v5.4s, v17.4s, v18.4s +; CHECK-GI-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-GI-NEXT: mov v25.s[3], w13 +; CHECK-GI-NEXT: add v6.4s, v7.4s, v23.4s +; CHECK-GI-NEXT: add v0.4s, v26.4s, v0.4s ; CHECK-GI-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-GI-NEXT: add v3.4s, v18.4s, v20.4s -; CHECK-GI-NEXT: add v0.4s, v11.4s, v0.4s +; CHECK-GI-NEXT: add v7.4s, v24.4s, v25.4s +; CHECK-GI-NEXT: add v3.4s, v5.4s, v6.4s ; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: add v0.4s, v21.4s, v0.4s +; CHECK-GI-NEXT: add v0.4s, v7.4s, v0.4s ; CHECK-GI-NEXT: fmov w8, s1 ; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s ; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w9, s0 ; CHECK-GI-NEXT: add w0, w8, w9 -; CHECK-GI-NEXT: ldp d11, d10, [sp], #48 // 16-byte Folded Reload +; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-GI-NEXT: ret entry: %az = sext <25 x i8> %a to <25 x i32> @@ -3972,197 +4541,412 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; ; CHECK-GI-LABEL: test_udot_v33i8: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 -; CHECK-GI-NEXT: .cfi_offset b8, -16 -; CHECK-GI-NEXT: ldp q21, q25, [x1] +; CHECK-GI-NEXT: sub sp, sp, #112 +; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 112 +; CHECK-GI-NEXT: .cfi_offset w19, -8 +; CHECK-GI-NEXT: .cfi_offset w20, -16 +; CHECK-GI-NEXT: .cfi_offset w21, -24 +; CHECK-GI-NEXT: .cfi_offset w22, -32 +; CHECK-GI-NEXT: .cfi_offset w23, -40 +; CHECK-GI-NEXT: .cfi_offset w24, -48 +; CHECK-GI-NEXT: .cfi_offset w25, -56 +; CHECK-GI-NEXT: .cfi_offset w26, -64 +; CHECK-GI-NEXT: .cfi_offset w27, -72 +; CHECK-GI-NEXT: .cfi_offset w28, -80 +; CHECK-GI-NEXT: .cfi_offset w30, -88 +; CHECK-GI-NEXT: .cfi_offset w29, -96 +; CHECK-GI-NEXT: ldp q7, q16, [x1] ; CHECK-GI-NEXT: fmov s5, wzr -; CHECK-GI-NEXT: ldp q26, q22, [x0] +; CHECK-GI-NEXT: str w2, [sp, #12] // 4-byte Folded Spill ; CHECK-GI-NEXT: fmov s6, wzr ; CHECK-GI-NEXT: fmov s0, wzr ; CHECK-GI-NEXT: fmov s1, wzr ; CHECK-GI-NEXT: fmov s3, wzr -; CHECK-GI-NEXT: umov w8, v21.b[0] -; CHECK-GI-NEXT: umov w9, v21.b[4] -; CHECK-GI-NEXT: umov w10, v21.b[1] -; CHECK-GI-NEXT: umov w13, v21.b[8] -; CHECK-GI-NEXT: umov w11, v21.b[5] -; CHECK-GI-NEXT: umov w14, v21.b[9] -; CHECK-GI-NEXT: umov w15, v25.b[0] -; CHECK-GI-NEXT: umov w12, v21.b[2] ; CHECK-GI-NEXT: fmov s2, wzr +; CHECK-GI-NEXT: mov b23, v7.b[7] +; CHECK-GI-NEXT: mov b17, v7.b[1] +; CHECK-GI-NEXT: fmov w11, s7 +; CHECK-GI-NEXT: mov b18, v7.b[2] +; CHECK-GI-NEXT: mov b19, v7.b[3] +; CHECK-GI-NEXT: mov b20, v7.b[4] +; CHECK-GI-NEXT: mov b21, v7.b[5] +; CHECK-GI-NEXT: mov b22, v7.b[6] +; CHECK-GI-NEXT: mov b24, v7.b[8] +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: mov b25, v7.b[9] +; CHECK-GI-NEXT: mov b26, v7.b[10] +; CHECK-GI-NEXT: mov b27, v7.b[11] +; CHECK-GI-NEXT: mov b28, v7.b[12] +; CHECK-GI-NEXT: mov b29, v7.b[13] +; CHECK-GI-NEXT: mov b30, v7.b[14] +; CHECK-GI-NEXT: mov b7, v7.b[15] +; CHECK-GI-NEXT: fmov w7, s23 +; CHECK-GI-NEXT: mov b23, v16.b[7] +; CHECK-GI-NEXT: fmov w10, s17 +; CHECK-GI-NEXT: fmov w9, s18 +; CHECK-GI-NEXT: fmov w13, s19 +; CHECK-GI-NEXT: fmov w8, s24 +; CHECK-GI-NEXT: mov b17, v16.b[2] +; CHECK-GI-NEXT: fmov w12, s20 +; CHECK-GI-NEXT: fmov w16, s25 +; CHECK-GI-NEXT: fmov w23, s21 +; CHECK-GI-NEXT: uxtb w10, w10 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov b18, v16.b[1] +; CHECK-GI-NEXT: stp s23, s7, [sp, #4] // 8-byte Folded Spill +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: fmov s7, w11 +; CHECK-GI-NEXT: fmov w5, s17 +; CHECK-GI-NEXT: fmov w27, s26 +; CHECK-GI-NEXT: mov b21, v16.b[5] +; CHECK-GI-NEXT: fmov s17, w8 +; CHECK-GI-NEXT: uxtb w8, w12 +; CHECK-GI-NEXT: fmov w20, s22 +; CHECK-GI-NEXT: mov v7.h[1], w10 +; CHECK-GI-NEXT: uxtb w10, w16 +; CHECK-GI-NEXT: mov b19, v16.b[3] +; CHECK-GI-NEXT: mov b22, v16.b[4] +; CHECK-GI-NEXT: mov b20, v16.b[6] +; CHECK-GI-NEXT: fmov w21, s27 +; CHECK-GI-NEXT: mov v17.h[1], w10 +; CHECK-GI-NEXT: fmov w24, s28 +; CHECK-GI-NEXT: mov b24, v16.b[8] +; CHECK-GI-NEXT: fmov w22, s29 +; CHECK-GI-NEXT: mov b26, v16.b[9] +; CHECK-GI-NEXT: fmov w4, s30 +; CHECK-GI-NEXT: uxtb w10, w21 +; CHECK-GI-NEXT: mov v7.h[2], w9 +; CHECK-GI-NEXT: uxtb w9, w13 +; CHECK-GI-NEXT: str s20, [sp] // 4-byte Folded Spill +; CHECK-GI-NEXT: mov b25, v16.b[10] +; CHECK-GI-NEXT: fmov w25, s18 +; CHECK-GI-NEXT: uxtb w22, w22 +; CHECK-GI-NEXT: mov b27, v16.b[11] +; CHECK-GI-NEXT: mov b28, v16.b[12] +; CHECK-GI-NEXT: mov b29, v16.b[13] +; CHECK-GI-NEXT: mov b30, v16.b[14] +; CHECK-GI-NEXT: fmov w26, s16 +; CHECK-GI-NEXT: mov v7.h[3], w9 +; CHECK-GI-NEXT: uxtb w9, w27 +; CHECK-GI-NEXT: mov b31, v16.b[15] +; CHECK-GI-NEXT: ldp q18, q16, [x0] +; CHECK-GI-NEXT: fmov w2, s21 +; CHECK-GI-NEXT: uxtb w26, w26 +; CHECK-GI-NEXT: mov v17.h[2], w9 +; CHECK-GI-NEXT: fmov w14, s22 +; CHECK-GI-NEXT: fmov w3, s25 +; CHECK-GI-NEXT: fmov w15, s19 +; CHECK-GI-NEXT: fmov w19, s24 +; CHECK-GI-NEXT: mov v7.h[4], w8 +; CHECK-GI-NEXT: uxtb w8, w23 +; CHECK-GI-NEXT: mov b21, v18.b[2] +; CHECK-GI-NEXT: mov b22, v18.b[1] +; CHECK-GI-NEXT: mov b25, v18.b[5] +; CHECK-GI-NEXT: mov b23, v18.b[6] +; CHECK-GI-NEXT: uxtb w19, w19 +; CHECK-GI-NEXT: uxtb w3, w3 +; CHECK-GI-NEXT: mov v17.h[3], w10 +; CHECK-GI-NEXT: uxtb w10, w24 +; CHECK-GI-NEXT: uxtb w24, w7 +; CHECK-GI-NEXT: mov b19, v18.b[3] +; CHECK-GI-NEXT: mov v7.h[5], w8 +; CHECK-GI-NEXT: uxtb w8, w20 +; CHECK-GI-NEXT: fmov w29, s21 +; CHECK-GI-NEXT: mov b21, v18.b[10] +; CHECK-GI-NEXT: fmov w9, s22 +; CHECK-GI-NEXT: fmov w6, s26 +; CHECK-GI-NEXT: mov v17.h[4], w10 +; CHECK-GI-NEXT: uxtb w10, w25 +; CHECK-GI-NEXT: fmov w17, s27 +; CHECK-GI-NEXT: mov b26, v18.b[4] +; CHECK-GI-NEXT: fmov w18, s28 +; CHECK-GI-NEXT: fmov w16, s29 +; CHECK-GI-NEXT: mov v7.h[6], w8 +; CHECK-GI-NEXT: fmov w8, s18 +; CHECK-GI-NEXT: mov b24, v18.b[7] +; CHECK-GI-NEXT: fmov w30, s21 +; CHECK-GI-NEXT: mov b20, v18.b[8] +; CHECK-GI-NEXT: mov b27, v18.b[9] +; CHECK-GI-NEXT: uxtb w16, w16 +; CHECK-GI-NEXT: mov b28, v18.b[11] +; CHECK-GI-NEXT: mov b29, v18.b[12] +; CHECK-GI-NEXT: fmov w23, s25 +; CHECK-GI-NEXT: mov b25, v18.b[13] +; CHECK-GI-NEXT: fmov w21, s23 +; CHECK-GI-NEXT: mov v7.h[7], w24 +; CHECK-GI-NEXT: uxtb w24, w8 +; CHECK-GI-NEXT: uxtb w8, w9 +; CHECK-GI-NEXT: uxtb w9, w29 +; CHECK-GI-NEXT: mov b23, v18.b[14] +; CHECK-GI-NEXT: mov b22, v18.b[15] +; CHECK-GI-NEXT: fmov s21, w24 +; CHECK-GI-NEXT: fmov s18, w26 +; CHECK-GI-NEXT: fmov w28, s19 +; CHECK-GI-NEXT: mov b19, v16.b[1] +; CHECK-GI-NEXT: mov v17.h[5], w22 +; CHECK-GI-NEXT: fmov w7, s20 +; CHECK-GI-NEXT: fmov w11, s27 +; CHECK-GI-NEXT: fmov w27, s26 +; CHECK-GI-NEXT: mov b20, v16.b[2] +; CHECK-GI-NEXT: mov v21.h[1], w8 +; CHECK-GI-NEXT: uxtb w8, w4 +; CHECK-GI-NEXT: mov v18.h[1], w10 +; CHECK-GI-NEXT: uxtb w10, w5 +; CHECK-GI-NEXT: uxtb w7, w7 +; CHECK-GI-NEXT: fmov w24, s23 +; CHECK-GI-NEXT: mov b23, v16.b[6] +; CHECK-GI-NEXT: fmov w4, s22 +; CHECK-GI-NEXT: mov b22, v16.b[8] +; CHECK-GI-NEXT: mov v17.h[6], w8 +; CHECK-GI-NEXT: fmov w8, s19 +; CHECK-GI-NEXT: fmov s19, w19 +; CHECK-GI-NEXT: mov v21.h[2], w9 +; CHECK-GI-NEXT: uxtb w9, w28 +; CHECK-GI-NEXT: mov v18.h[2], w10 +; CHECK-GI-NEXT: uxtb w10, w6 +; CHECK-GI-NEXT: mov b27, v16.b[9] +; CHECK-GI-NEXT: fmov w20, s24 +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: mov b24, v16.b[3] +; CHECK-GI-NEXT: fmov w5, s20 +; CHECK-GI-NEXT: mov v19.h[1], w10 +; CHECK-GI-NEXT: fmov w10, s23 +; CHECK-GI-NEXT: fmov s23, w7 +; CHECK-GI-NEXT: mov v21.h[3], w9 +; CHECK-GI-NEXT: uxtb w9, w11 +; CHECK-GI-NEXT: uxtb w11, w27 +; CHECK-GI-NEXT: uxtb w27, w30 +; CHECK-GI-NEXT: uxtb w5, w5 +; CHECK-GI-NEXT: fmov w7, s22 +; CHECK-GI-NEXT: uxtb w10, w10 +; CHECK-GI-NEXT: mov v23.h[1], w9 +; CHECK-GI-NEXT: fmov w9, s16 +; CHECK-GI-NEXT: mov b20, v16.b[10] +; CHECK-GI-NEXT: fmov w22, s28 +; CHECK-GI-NEXT: fmov w25, s25 +; CHECK-GI-NEXT: uxtb w7, w7 +; CHECK-GI-NEXT: mov v21.h[4], w11 +; CHECK-GI-NEXT: fmov w11, s27 +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov b25, v16.b[5] +; CHECK-GI-NEXT: fmov w29, s24 +; CHECK-GI-NEXT: fmov s22, w7 +; CHECK-GI-NEXT: mov v23.h[2], w27 +; CHECK-GI-NEXT: mov b24, v16.b[11] +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: fmov w27, s20 +; CHECK-GI-NEXT: fmov s20, w9 +; CHECK-GI-NEXT: fmov w26, s29 +; CHECK-GI-NEXT: mov b26, v16.b[4] +; CHECK-GI-NEXT: mov v19.h[2], w3 +; CHECK-GI-NEXT: uxtb w3, w29 +; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v22.h[1], w11 +; CHECK-GI-NEXT: uxtb w11, w15 +; CHECK-GI-NEXT: uxtb w15, w22 +; CHECK-GI-NEXT: uxtb w22, w23 +; CHECK-GI-NEXT: mov v20.h[1], w8 +; CHECK-GI-NEXT: fmov w6, s25 +; CHECK-GI-NEXT: mov v18.h[3], w11 +; CHECK-GI-NEXT: uxtb w11, w27 +; CHECK-GI-NEXT: mov v23.h[3], w15 +; CHECK-GI-NEXT: uxtb w15, w17 +; CHECK-GI-NEXT: uxtb w17, w21 +; CHECK-GI-NEXT: mov b25, v16.b[12] +; CHECK-GI-NEXT: fmov w28, s24 +; CHECK-GI-NEXT: mov v21.h[5], w22 +; CHECK-GI-NEXT: mov v22.h[2], w11 +; CHECK-GI-NEXT: uxtb w11, w14 +; CHECK-GI-NEXT: uxtb w14, w26 +; CHECK-GI-NEXT: mov v20.h[2], w5 +; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload +; CHECK-GI-NEXT: fmov w19, s26 +; CHECK-GI-NEXT: mov v18.h[4], w11 +; CHECK-GI-NEXT: uxtb w11, w28 +; CHECK-GI-NEXT: mov v23.h[4], w14 +; CHECK-GI-NEXT: uxtb w14, w25 +; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov b26, v16.b[13] +; CHECK-GI-NEXT: fmov w7, s25 +; CHECK-GI-NEXT: mov v19.h[3], w15 +; CHECK-GI-NEXT: uxtb w15, w18 +; CHECK-GI-NEXT: uxtb w18, w19 +; CHECK-GI-NEXT: mov v21.h[6], w17 +; CHECK-GI-NEXT: uxtb w17, w20 +; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v22.h[3], w11 +; CHECK-GI-NEXT: uxtb w11, w2 +; CHECK-GI-NEXT: mov v20.h[3], w3 +; CHECK-GI-NEXT: mov v23.h[5], w14 +; CHECK-GI-NEXT: uxtb w14, w24 +; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v18.h[5], w11 +; CHECK-GI-NEXT: uxtb w11, w7 +; CHECK-GI-NEXT: fmov w8, s26 +; CHECK-GI-NEXT: mov v19.h[4], w15 +; CHECK-GI-NEXT: ldr w15, [sp] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v21.h[7], w17 +; CHECK-GI-NEXT: uxtb w17, w6 +; CHECK-GI-NEXT: mov v22.h[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #8] // 4-byte Folded Reload +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: uxtb w15, w15 +; CHECK-GI-NEXT: fmov w13, s30 +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: mov v20.h[4], w18 +; CHECK-GI-NEXT: mov v23.h[6], w14 +; CHECK-GI-NEXT: mov v19.h[5], w16 +; CHECK-GI-NEXT: mov b27, v16.b[14] +; CHECK-GI-NEXT: mul v24.8h, v7.8h, v21.8h +; CHECK-GI-NEXT: mov v22.h[5], w8 +; CHECK-GI-NEXT: uxtb w8, w4 +; CHECK-GI-NEXT: mov b7, v16.b[7] +; CHECK-GI-NEXT: mov b16, v16.b[15] +; CHECK-GI-NEXT: fmov w12, s31 +; CHECK-GI-NEXT: mov v17.h[7], w11 +; CHECK-GI-NEXT: uxtb w11, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #4] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v20.h[5], w17 +; CHECK-GI-NEXT: mov v23.h[7], w8 +; CHECK-GI-NEXT: fmov w9, s27 +; CHECK-GI-NEXT: mov v18.h[6], w15 +; CHECK-GI-NEXT: uxtb w8, w12 +; CHECK-GI-NEXT: uxtb w13, w13 +; CHECK-GI-NEXT: mov v19.h[6], w11 +; CHECK-GI-NEXT: fmov w12, s16 +; CHECK-GI-NEXT: fmov w11, s7 ; CHECK-GI-NEXT: fmov s4, wzr +; CHECK-GI-NEXT: uxtb w9, w9 +; CHECK-GI-NEXT: mov v20.h[6], w10 +; CHECK-GI-NEXT: umov w10, v24.h[0] +; CHECK-GI-NEXT: mul v21.8h, v17.8h, v23.8h +; CHECK-GI-NEXT: mov v18.h[7], w13 ; CHECK-GI-NEXT: mov v5.s[1], wzr +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: mov v19.h[7], w8 +; CHECK-GI-NEXT: uxtb w8, w12 +; CHECK-GI-NEXT: umov w12, v24.h[4] +; CHECK-GI-NEXT: mov v22.h[6], w9 +; CHECK-GI-NEXT: umov w9, v24.h[1] +; CHECK-GI-NEXT: mov v20.h[7], w11 +; CHECK-GI-NEXT: umov w11, v24.h[5] +; CHECK-GI-NEXT: fmov s7, w10 +; CHECK-GI-NEXT: ldrb w10, [x1, #32] +; CHECK-GI-NEXT: umov w13, v21.h[0] +; CHECK-GI-NEXT: umov w14, v21.h[1] +; CHECK-GI-NEXT: umov w15, v21.h[4] ; CHECK-GI-NEXT: mov v6.s[1], wzr -; CHECK-GI-NEXT: fmov s7, w8 -; CHECK-GI-NEXT: fmov s17, w9 -; CHECK-GI-NEXT: umov w8, v21.b[6] -; CHECK-GI-NEXT: fmov s16, w13 -; CHECK-GI-NEXT: umov w9, v21.b[3] -; CHECK-GI-NEXT: umov w13, v21.b[7] -; CHECK-GI-NEXT: fmov s18, w15 -; CHECK-GI-NEXT: umov w15, v25.b[4] ; CHECK-GI-NEXT: mov v0.s[1], wzr -; CHECK-GI-NEXT: mov v7.s[1], w10 -; CHECK-GI-NEXT: umov w10, v21.b[12] -; CHECK-GI-NEXT: mov v17.s[1], w11 -; CHECK-GI-NEXT: umov w11, v21.b[13] -; CHECK-GI-NEXT: mov v16.s[1], w14 -; CHECK-GI-NEXT: umov w14, v25.b[1] +; CHECK-GI-NEXT: fmov s16, w12 +; CHECK-GI-NEXT: mov v22.h[7], w8 +; CHECK-GI-NEXT: umov w12, v24.h[6] +; CHECK-GI-NEXT: umov w8, v24.h[2] +; CHECK-GI-NEXT: mov v7.s[1], w9 +; CHECK-GI-NEXT: ldrb w9, [x0, #32] +; CHECK-GI-NEXT: fmov s17, w13 +; CHECK-GI-NEXT: mul v23.8h, v18.8h, v20.8h +; CHECK-GI-NEXT: umov w13, v24.h[7] +; CHECK-GI-NEXT: mov v16.s[1], w11 +; CHECK-GI-NEXT: umov w11, v21.h[5] +; CHECK-GI-NEXT: fmov s18, w15 +; CHECK-GI-NEXT: mul v19.8h, v19.8h, v22.8h +; CHECK-GI-NEXT: umov w15, v21.h[6] ; CHECK-GI-NEXT: mov v1.s[1], wzr +; CHECK-GI-NEXT: mov v17.s[1], w14 +; CHECK-GI-NEXT: umov w14, v21.h[2] +; CHECK-GI-NEXT: mov v7.s[2], w8 +; CHECK-GI-NEXT: mul w8, w10, w9 +; CHECK-GI-NEXT: umov w9, v23.h[0] +; CHECK-GI-NEXT: umov w10, v23.h[1] +; CHECK-GI-NEXT: mov v16.s[2], w12 +; CHECK-GI-NEXT: umov w12, v21.h[3] +; CHECK-GI-NEXT: mov v18.s[1], w11 +; CHECK-GI-NEXT: umov w11, v23.h[4] ; CHECK-GI-NEXT: mov v3.s[1], wzr ; CHECK-GI-NEXT: mov v2.s[1], wzr -; CHECK-GI-NEXT: fmov s20, w15 -; CHECK-GI-NEXT: umov w15, v25.b[13] +; CHECK-GI-NEXT: mov v17.s[2], w14 +; CHECK-GI-NEXT: umov w14, v23.h[5] ; CHECK-GI-NEXT: mov v4.s[1], wzr -; CHECK-GI-NEXT: fmov s19, w10 -; CHECK-GI-NEXT: mov v7.s[2], w12 -; CHECK-GI-NEXT: umov w12, v21.b[10] -; CHECK-GI-NEXT: mov v18.s[1], w14 -; CHECK-GI-NEXT: umov w14, v25.b[5] -; CHECK-GI-NEXT: mov v17.s[2], w8 -; CHECK-GI-NEXT: umov w8, v21.b[11] -; CHECK-GI-NEXT: umov w10, v21.b[14] +; CHECK-GI-NEXT: fmov s20, w9 +; CHECK-GI-NEXT: umov w9, v19.h[1] ; CHECK-GI-NEXT: mov v5.s[2], wzr -; CHECK-GI-NEXT: mov v19.s[1], w11 -; CHECK-GI-NEXT: umov w11, v25.b[2] +; CHECK-GI-NEXT: mov v16.s[3], w13 +; CHECK-GI-NEXT: umov w13, v19.h[0] +; CHECK-GI-NEXT: mov v18.s[2], w15 +; CHECK-GI-NEXT: umov w15, v21.h[7] +; CHECK-GI-NEXT: fmov s21, w11 +; CHECK-GI-NEXT: umov w11, v23.h[2] +; CHECK-GI-NEXT: mov v17.s[3], w12 +; CHECK-GI-NEXT: umov w12, v19.h[4] +; CHECK-GI-NEXT: mov v20.s[1], w10 +; CHECK-GI-NEXT: umov w10, v23.h[3] ; CHECK-GI-NEXT: mov v6.s[2], wzr -; CHECK-GI-NEXT: mov v16.s[2], w12 -; CHECK-GI-NEXT: umov w12, v25.b[8] -; CHECK-GI-NEXT: mov v7.s[3], w9 -; CHECK-GI-NEXT: mov v20.s[1], w14 -; CHECK-GI-NEXT: umov w14, v21.b[15] -; CHECK-GI-NEXT: umov w9, v25.b[9] -; CHECK-GI-NEXT: mov v17.s[3], w13 -; CHECK-GI-NEXT: umov w13, v25.b[12] +; CHECK-GI-NEXT: umov w16, v24.h[3] +; CHECK-GI-NEXT: fmov s22, w13 +; CHECK-GI-NEXT: umov w13, v19.h[5] +; CHECK-GI-NEXT: mov v21.s[1], w14 +; CHECK-GI-NEXT: umov w14, v23.h[6] +; CHECK-GI-NEXT: mov v18.s[3], w15 +; CHECK-GI-NEXT: umov w15, v19.h[2] +; CHECK-GI-NEXT: mov v20.s[2], w11 +; CHECK-GI-NEXT: umov w11, v19.h[6] ; CHECK-GI-NEXT: mov v0.s[2], wzr -; CHECK-GI-NEXT: mov v18.s[2], w11 -; CHECK-GI-NEXT: umov w11, v26.b[0] -; CHECK-GI-NEXT: mov v19.s[2], w10 -; CHECK-GI-NEXT: fmov s21, w12 -; CHECK-GI-NEXT: umov w12, v26.b[1] -; CHECK-GI-NEXT: mov v16.s[3], w8 -; CHECK-GI-NEXT: umov w8, v26.b[5] -; CHECK-GI-NEXT: umov w10, v25.b[6] +; CHECK-GI-NEXT: mov v22.s[1], w9 +; CHECK-GI-NEXT: umov w9, v23.h[7] +; CHECK-GI-NEXT: fmov s23, w12 +; CHECK-GI-NEXT: umov w12, v19.h[3] ; CHECK-GI-NEXT: mov v1.s[2], wzr -; CHECK-GI-NEXT: fmov s23, w13 -; CHECK-GI-NEXT: umov w13, v25.b[3] ; CHECK-GI-NEXT: mov v3.s[2], wzr -; CHECK-GI-NEXT: fmov s24, w11 -; CHECK-GI-NEXT: mov v21.s[1], w9 -; CHECK-GI-NEXT: umov w9, v25.b[10] -; CHECK-GI-NEXT: umov w11, v26.b[2] -; CHECK-GI-NEXT: mov v19.s[3], w14 -; CHECK-GI-NEXT: umov w14, v26.b[13] -; CHECK-GI-NEXT: mov v23.s[1], w15 -; CHECK-GI-NEXT: umov w15, v25.b[14] -; CHECK-GI-NEXT: mov v20.s[2], w10 -; CHECK-GI-NEXT: mov v24.s[1], w12 -; CHECK-GI-NEXT: umov w12, v26.b[4] -; CHECK-GI-NEXT: umov w10, v25.b[7] -; CHECK-GI-NEXT: mov v21.s[2], w9 -; CHECK-GI-NEXT: umov w9, v25.b[11] -; CHECK-GI-NEXT: mov v18.s[3], w13 -; CHECK-GI-NEXT: umov w13, v26.b[9] +; CHECK-GI-NEXT: mov v21.s[2], w14 ; CHECK-GI-NEXT: mov v2.s[2], wzr ; CHECK-GI-NEXT: mov v4.s[2], wzr -; CHECK-GI-NEXT: mov v23.s[2], w15 -; CHECK-GI-NEXT: umov w15, v25.b[15] +; CHECK-GI-NEXT: mov v23.s[1], w13 ; CHECK-GI-NEXT: mov v5.s[3], wzr -; CHECK-GI-NEXT: fmov s27, w12 -; CHECK-GI-NEXT: mov v24.s[2], w11 -; CHECK-GI-NEXT: umov w11, v26.b[6] -; CHECK-GI-NEXT: umov w12, v26.b[8] -; CHECK-GI-NEXT: mov v21.s[3], w9 -; CHECK-GI-NEXT: umov w9, v26.b[12] -; CHECK-GI-NEXT: mov v20.s[3], w10 -; CHECK-GI-NEXT: umov w10, v26.b[3] ; CHECK-GI-NEXT: mov v6.s[3], wzr -; CHECK-GI-NEXT: mov v27.s[1], w8 -; CHECK-GI-NEXT: mov v23.s[3], w15 -; CHECK-GI-NEXT: umov w15, v22.b[0] -; CHECK-GI-NEXT: umov w8, v26.b[7] +; CHECK-GI-NEXT: mov v22.s[2], w15 +; CHECK-GI-NEXT: mov v7.s[3], w16 +; CHECK-GI-NEXT: mov v20.s[3], w10 ; CHECK-GI-NEXT: mov v0.s[3], wzr ; CHECK-GI-NEXT: mov v1.s[3], wzr -; CHECK-GI-NEXT: fmov s25, w12 -; CHECK-GI-NEXT: fmov s29, w9 -; CHECK-GI-NEXT: umov w9, v22.b[5] -; CHECK-GI-NEXT: mov v24.s[3], w10 -; CHECK-GI-NEXT: umov w10, v22.b[1] -; CHECK-GI-NEXT: umov w12, v26.b[10] -; CHECK-GI-NEXT: mov v27.s[2], w11 -; CHECK-GI-NEXT: umov w11, v22.b[4] -; CHECK-GI-NEXT: fmov s28, w15 -; CHECK-GI-NEXT: mov v25.s[1], w13 -; CHECK-GI-NEXT: umov w13, v26.b[14] -; CHECK-GI-NEXT: mov v29.s[1], w14 -; CHECK-GI-NEXT: umov w15, v22.b[12] -; CHECK-GI-NEXT: umov w14, v22.b[2] ; CHECK-GI-NEXT: mov v3.s[3], wzr -; CHECK-GI-NEXT: mov v28.s[1], w10 -; CHECK-GI-NEXT: umov w10, v22.b[13] +; CHECK-GI-NEXT: mov v21.s[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #12] // 4-byte Folded Reload ; CHECK-GI-NEXT: mov v2.s[3], wzr -; CHECK-GI-NEXT: fmov s30, w11 -; CHECK-GI-NEXT: umov w11, v22.b[6] -; CHECK-GI-NEXT: mov v27.s[3], w8 -; CHECK-GI-NEXT: mov v25.s[2], w12 -; CHECK-GI-NEXT: mov v29.s[2], w13 -; CHECK-GI-NEXT: umov w13, v26.b[11] -; CHECK-GI-NEXT: fmov s31, w15 -; CHECK-GI-NEXT: umov w15, v26.b[15] -; CHECK-GI-NEXT: umov w12, v22.b[9] -; CHECK-GI-NEXT: mov v30.s[1], w9 -; CHECK-GI-NEXT: umov w9, v22.b[8] -; CHECK-GI-NEXT: mov v28.s[2], w14 -; CHECK-GI-NEXT: ldrb w14, [x1, #32] -; CHECK-GI-NEXT: umov w8, v22.b[15] -; CHECK-GI-NEXT: mul v17.4s, v17.4s, v27.4s -; CHECK-GI-NEXT: mov v31.s[1], w10 -; CHECK-GI-NEXT: umov w10, v22.b[14] -; CHECK-GI-NEXT: mov v25.s[3], w13 -; CHECK-GI-NEXT: ldrb w13, [x0, #32] -; CHECK-GI-NEXT: mov v29.s[3], w15 +; CHECK-GI-NEXT: mov v23.s[2], w11 +; CHECK-GI-NEXT: umov w11, v19.h[7] +; CHECK-GI-NEXT: fmov s19, w8 +; CHECK-GI-NEXT: mov v22.s[3], w12 ; CHECK-GI-NEXT: mov v4.s[3], wzr -; CHECK-GI-NEXT: mov v30.s[2], w11 -; CHECK-GI-NEXT: fmov s26, w9 -; CHECK-GI-NEXT: umov w9, v22.b[7] -; CHECK-GI-NEXT: umov w11, v22.b[3] ; CHECK-GI-NEXT: add v5.4s, v5.4s, v6.4s -; CHECK-GI-NEXT: mla v17.4s, v7.4s, v24.4s -; CHECK-GI-NEXT: mov v31.s[2], w10 +; CHECK-GI-NEXT: add v6.4s, v7.4s, v16.4s +; CHECK-GI-NEXT: add v7.4s, v17.4s, v18.4s ; CHECK-GI-NEXT: add v1.4s, v1.4s, v3.4s -; CHECK-GI-NEXT: mov v26.s[1], w12 -; CHECK-GI-NEXT: umov w12, v22.b[10] -; CHECK-GI-NEXT: mul v19.4s, v19.4s, v29.4s -; CHECK-GI-NEXT: mov v30.s[3], w9 -; CHECK-GI-NEXT: mul w9, w14, w13 -; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s -; CHECK-GI-NEXT: mov v28.s[3], w11 +; CHECK-GI-NEXT: mov v19.s[1], wzr +; CHECK-GI-NEXT: add v16.4s, v20.4s, v21.4s +; CHECK-GI-NEXT: mov v23.s[3], w11 ; CHECK-GI-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-GI-NEXT: mov v31.s[3], w8 -; CHECK-GI-NEXT: umov w8, v22.b[11] -; CHECK-GI-NEXT: fmov s8, w9 -; CHECK-GI-NEXT: mov v26.s[2], w12 -; CHECK-GI-NEXT: mla v19.4s, v16.4s, v25.4s -; CHECK-GI-NEXT: mul v20.4s, v20.4s, v30.4s +; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s +; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: mov v19.s[2], wzr +; CHECK-GI-NEXT: add v17.4s, v22.4s, v23.4s ; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s -; CHECK-GI-NEXT: mov v8.s[1], wzr -; CHECK-GI-NEXT: mul v22.4s, v23.4s, v31.4s -; CHECK-GI-NEXT: mov v26.s[3], w8 -; CHECK-GI-NEXT: add v3.4s, v17.4s, v19.4s -; CHECK-GI-NEXT: mla v20.4s, v18.4s, v28.4s -; CHECK-GI-NEXT: mov v8.s[2], wzr -; CHECK-GI-NEXT: mla v22.4s, v21.4s, v26.4s -; CHECK-GI-NEXT: mov v8.s[3], wzr -; CHECK-GI-NEXT: add v4.4s, v20.4s, v22.4s -; CHECK-GI-NEXT: add v0.4s, v8.4s, v0.4s +; CHECK-GI-NEXT: mov v19.s[3], wzr +; CHECK-GI-NEXT: add v4.4s, v16.4s, v17.4s ; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: add v0.4s, v19.4s, v0.4s ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: add sp, sp, #112 ; CHECK-GI-NEXT: ret entry: %0 = load <33 x i8>, ptr %a @@ -4359,197 +5143,412 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; ; CHECK-GI-LABEL: test_sdot_v33i8: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 -; CHECK-GI-NEXT: .cfi_offset b8, -16 -; CHECK-GI-NEXT: ldp q21, q25, [x1] -; CHECK-GI-NEXT: fmov s5, wzr -; CHECK-GI-NEXT: ldp q26, q22, [x0] -; CHECK-GI-NEXT: fmov s6, wzr -; CHECK-GI-NEXT: fmov s0, wzr +; CHECK-GI-NEXT: sub sp, sp, #112 +; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 112 +; CHECK-GI-NEXT: .cfi_offset w19, -8 +; CHECK-GI-NEXT: .cfi_offset w20, -16 +; CHECK-GI-NEXT: .cfi_offset w21, -24 +; CHECK-GI-NEXT: .cfi_offset w22, -32 +; CHECK-GI-NEXT: .cfi_offset w23, -40 +; CHECK-GI-NEXT: .cfi_offset w24, -48 +; CHECK-GI-NEXT: .cfi_offset w25, -56 +; CHECK-GI-NEXT: .cfi_offset w26, -64 +; CHECK-GI-NEXT: .cfi_offset w27, -72 +; CHECK-GI-NEXT: .cfi_offset w28, -80 +; CHECK-GI-NEXT: .cfi_offset w30, -88 +; CHECK-GI-NEXT: .cfi_offset w29, -96 +; CHECK-GI-NEXT: ldp q7, q16, [x1] ; CHECK-GI-NEXT: fmov s1, wzr +; CHECK-GI-NEXT: str w2, [sp, #12] // 4-byte Folded Spill ; CHECK-GI-NEXT: fmov s3, wzr -; CHECK-GI-NEXT: smov w8, v21.b[0] -; CHECK-GI-NEXT: smov w9, v21.b[4] -; CHECK-GI-NEXT: smov w10, v21.b[1] -; CHECK-GI-NEXT: smov w13, v21.b[8] -; CHECK-GI-NEXT: smov w11, v21.b[5] -; CHECK-GI-NEXT: smov w14, v21.b[9] -; CHECK-GI-NEXT: smov w15, v25.b[0] -; CHECK-GI-NEXT: smov w12, v21.b[2] ; CHECK-GI-NEXT: fmov s2, wzr +; CHECK-GI-NEXT: fmov s5, wzr ; CHECK-GI-NEXT: fmov s4, wzr -; CHECK-GI-NEXT: mov v5.s[1], wzr -; CHECK-GI-NEXT: mov v6.s[1], wzr -; CHECK-GI-NEXT: fmov s7, w8 -; CHECK-GI-NEXT: fmov s17, w9 -; CHECK-GI-NEXT: smov w8, v21.b[6] -; CHECK-GI-NEXT: fmov s16, w13 -; CHECK-GI-NEXT: smov w9, v21.b[3] -; CHECK-GI-NEXT: smov w13, v21.b[7] -; CHECK-GI-NEXT: fmov s18, w15 -; CHECK-GI-NEXT: smov w15, v25.b[4] -; CHECK-GI-NEXT: mov v0.s[1], wzr -; CHECK-GI-NEXT: mov v7.s[1], w10 -; CHECK-GI-NEXT: smov w10, v21.b[12] -; CHECK-GI-NEXT: mov v17.s[1], w11 -; CHECK-GI-NEXT: smov w11, v21.b[13] -; CHECK-GI-NEXT: mov v16.s[1], w14 -; CHECK-GI-NEXT: smov w14, v25.b[1] +; CHECK-GI-NEXT: fmov s6, wzr +; CHECK-GI-NEXT: mov b19, v7.b[3] +; CHECK-GI-NEXT: mov b23, v7.b[7] +; CHECK-GI-NEXT: mov b17, v7.b[1] +; CHECK-GI-NEXT: fmov w11, s7 +; CHECK-GI-NEXT: mov b18, v7.b[2] +; CHECK-GI-NEXT: mov b20, v7.b[4] +; CHECK-GI-NEXT: mov b21, v7.b[5] +; CHECK-GI-NEXT: mov b22, v7.b[6] +; CHECK-GI-NEXT: mov b24, v7.b[8] +; CHECK-GI-NEXT: mov b25, v7.b[9] +; CHECK-GI-NEXT: mov b26, v7.b[10] +; CHECK-GI-NEXT: mov b27, v7.b[11] +; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: mov b28, v7.b[12] +; CHECK-GI-NEXT: fmov w14, s19 +; CHECK-GI-NEXT: mov b19, v7.b[13] +; CHECK-GI-NEXT: mov b29, v7.b[14] +; CHECK-GI-NEXT: mov b7, v7.b[15] +; CHECK-GI-NEXT: fmov w7, s23 +; CHECK-GI-NEXT: mov b23, v16.b[6] +; CHECK-GI-NEXT: fmov w10, s17 +; CHECK-GI-NEXT: fmov w9, s18 +; CHECK-GI-NEXT: fmov w8, s24 +; CHECK-GI-NEXT: mov b30, v16.b[1] +; CHECK-GI-NEXT: fmov w16, s25 +; CHECK-GI-NEXT: fmov w12, s20 +; CHECK-GI-NEXT: fmov w24, s21 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: sxtb w7, w7 +; CHECK-GI-NEXT: fmov w22, s22 +; CHECK-GI-NEXT: stp s23, s7, [sp, #4] // 8-byte Folded Spill +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: fmov s7, w11 +; CHECK-GI-NEXT: mov b20, v16.b[2] +; CHECK-GI-NEXT: mov b17, v16.b[3] +; CHECK-GI-NEXT: mov b21, v16.b[4] +; CHECK-GI-NEXT: mov b18, v16.b[5] +; CHECK-GI-NEXT: fmov w27, s26 +; CHECK-GI-NEXT: fmov w25, s27 +; CHECK-GI-NEXT: mov b22, v16.b[7] +; CHECK-GI-NEXT: fmov w26, s28 +; CHECK-GI-NEXT: mov v7.h[1], w10 +; CHECK-GI-NEXT: sxtb w10, w16 +; CHECK-GI-NEXT: mov b25, v16.b[8] +; CHECK-GI-NEXT: fmov w23, s19 +; CHECK-GI-NEXT: mov b24, v16.b[9] +; CHECK-GI-NEXT: fmov w5, s29 +; CHECK-GI-NEXT: mov b26, v16.b[10] +; CHECK-GI-NEXT: mov b19, v16.b[11] +; CHECK-GI-NEXT: fmov w6, s30 +; CHECK-GI-NEXT: mov b27, v16.b[12] +; CHECK-GI-NEXT: mov b28, v16.b[13] +; CHECK-GI-NEXT: mov b29, v16.b[14] +; CHECK-GI-NEXT: sxtb w30, w23 +; CHECK-GI-NEXT: sxtb w5, w5 +; CHECK-GI-NEXT: mov v7.h[2], w9 +; CHECK-GI-NEXT: sxtb w9, w14 +; CHECK-GI-NEXT: fmov w20, s16 +; CHECK-GI-NEXT: mov b30, v16.b[15] +; CHECK-GI-NEXT: fmov s16, w8 +; CHECK-GI-NEXT: sxtb w8, w12 +; CHECK-GI-NEXT: fmov w15, s17 +; CHECK-GI-NEXT: fmov w11, s18 +; CHECK-GI-NEXT: ldp q18, q17, [x0] +; CHECK-GI-NEXT: mov v7.h[3], w9 +; CHECK-GI-NEXT: sxtb w9, w27 +; CHECK-GI-NEXT: fmov w18, s20 +; CHECK-GI-NEXT: sxtb w15, w15 +; CHECK-GI-NEXT: mov v16.h[1], w10 +; CHECK-GI-NEXT: sxtb w10, w25 +; CHECK-GI-NEXT: mov b20, v18.b[3] +; CHECK-GI-NEXT: fmov w2, s22 +; CHECK-GI-NEXT: mov b22, v18.b[1] +; CHECK-GI-NEXT: sxtb w18, w18 +; CHECK-GI-NEXT: fmov w13, s21 +; CHECK-GI-NEXT: mov b21, v18.b[2] +; CHECK-GI-NEXT: mov v7.h[4], w8 +; CHECK-GI-NEXT: fmov w3, s19 +; CHECK-GI-NEXT: mov b19, v18.b[6] +; CHECK-GI-NEXT: mov v16.h[2], w9 +; CHECK-GI-NEXT: sxtb w9, w24 +; CHECK-GI-NEXT: fmov w21, s25 +; CHECK-GI-NEXT: sxtb w13, w13 +; CHECK-GI-NEXT: fmov w28, s20 +; CHECK-GI-NEXT: mov b20, v18.b[11] +; CHECK-GI-NEXT: fmov w8, s22 +; CHECK-GI-NEXT: mov b25, v18.b[8] +; CHECK-GI-NEXT: fmov w29, s21 +; CHECK-GI-NEXT: mov v7.h[5], w9 +; CHECK-GI-NEXT: sxtb w9, w22 +; CHECK-GI-NEXT: fmov w19, s24 +; CHECK-GI-NEXT: mov v16.h[3], w10 +; CHECK-GI-NEXT: sxtb w10, w26 +; CHECK-GI-NEXT: fmov w26, s18 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: sxtb w29, w29 +; CHECK-GI-NEXT: mov b24, v18.b[4] +; CHECK-GI-NEXT: mov b23, v18.b[5] +; CHECK-GI-NEXT: fmov w17, s27 +; CHECK-GI-NEXT: mov b27, v18.b[9] +; CHECK-GI-NEXT: sxtb w23, w26 +; CHECK-GI-NEXT: mov v7.h[6], w9 +; CHECK-GI-NEXT: fmov w24, s19 +; CHECK-GI-NEXT: mov v16.h[4], w10 +; CHECK-GI-NEXT: mov b19, v18.b[14] +; CHECK-GI-NEXT: fmov w10, s25 +; CHECK-GI-NEXT: fmov w4, s26 +; CHECK-GI-NEXT: fmov w16, s28 +; CHECK-GI-NEXT: mov b26, v18.b[7] +; CHECK-GI-NEXT: mov b28, v18.b[10] +; CHECK-GI-NEXT: fmov w27, s24 +; CHECK-GI-NEXT: mov b24, v18.b[12] +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v7.h[7], w7 +; CHECK-GI-NEXT: fmov w7, s20 +; CHECK-GI-NEXT: sxtb w4, w4 +; CHECK-GI-NEXT: fmov s20, w23 +; CHECK-GI-NEXT: fmov w25, s23 +; CHECK-GI-NEXT: mov b23, v18.b[13] +; CHECK-GI-NEXT: mov b22, v18.b[15] +; CHECK-GI-NEXT: mov v16.h[5], w30 +; CHECK-GI-NEXT: sxtb w7, w7 +; CHECK-GI-NEXT: fmov w9, s27 +; CHECK-GI-NEXT: mov b21, v17.b[1] +; CHECK-GI-NEXT: mov v20.h[1], w8 +; CHECK-GI-NEXT: sxtb w8, w20 +; CHECK-GI-NEXT: sxtb w20, w6 +; CHECK-GI-NEXT: fmov w6, s19 +; CHECK-GI-NEXT: fmov w26, s28 +; CHECK-GI-NEXT: mov b28, v17.b[8] +; CHECK-GI-NEXT: fmov s18, w8 +; CHECK-GI-NEXT: sxtb w8, w21 +; CHECK-GI-NEXT: mov v16.h[6], w5 +; CHECK-GI-NEXT: fmov w5, s22 +; CHECK-GI-NEXT: fmov s22, w10 +; CHECK-GI-NEXT: sxtb w10, w27 +; CHECK-GI-NEXT: sxtb w26, w26 +; CHECK-GI-NEXT: mov v20.h[2], w29 +; CHECK-GI-NEXT: fmov s19, w8 +; CHECK-GI-NEXT: sxtb w8, w28 +; CHECK-GI-NEXT: sxtb w28, w19 +; CHECK-GI-NEXT: sxtb w19, w9 +; CHECK-GI-NEXT: fmov w27, s17 +; CHECK-GI-NEXT: mov b25, v17.b[2] +; CHECK-GI-NEXT: fmov w29, s21 +; CHECK-GI-NEXT: mov b21, v17.b[9] +; CHECK-GI-NEXT: mov v22.h[1], w19 +; CHECK-GI-NEXT: fmov w23, s23 +; CHECK-GI-NEXT: mov v20.h[3], w8 +; CHECK-GI-NEXT: mov b23, v17.b[6] +; CHECK-GI-NEXT: fmov w30, s24 +; CHECK-GI-NEXT: sxtb w27, w27 +; CHECK-GI-NEXT: mov b24, v17.b[5] +; CHECK-GI-NEXT: mov v18.h[1], w20 +; CHECK-GI-NEXT: fmov w21, s25 +; CHECK-GI-NEXT: mov b25, v17.b[10] +; CHECK-GI-NEXT: mov v19.h[1], w28 +; CHECK-GI-NEXT: sxtb w28, w29 +; CHECK-GI-NEXT: mov v22.h[2], w26 +; CHECK-GI-NEXT: fmov w26, s21 +; CHECK-GI-NEXT: mov v20.h[4], w10 +; CHECK-GI-NEXT: fmov w10, s28 +; CHECK-GI-NEXT: fmov s21, w27 +; CHECK-GI-NEXT: sxtb w21, w21 +; CHECK-GI-NEXT: mov b27, v17.b[3] +; CHECK-GI-NEXT: fmov w19, s23 +; CHECK-GI-NEXT: sxtb w26, w26 +; CHECK-GI-NEXT: fmov w22, s26 +; CHECK-GI-NEXT: mov b26, v17.b[4] +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v21.h[1], w28 +; CHECK-GI-NEXT: fmov w8, s24 +; CHECK-GI-NEXT: mov b24, v17.b[11] +; CHECK-GI-NEXT: fmov w27, s25 +; CHECK-GI-NEXT: mov v18.h[2], w18 +; CHECK-GI-NEXT: sxtb w18, w25 +; CHECK-GI-NEXT: fmov s23, w10 +; CHECK-GI-NEXT: fmov w20, s27 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: fmov w9, s26 +; CHECK-GI-NEXT: mov b26, v17.b[12] +; CHECK-GI-NEXT: sxtb w25, w27 +; CHECK-GI-NEXT: mov v20.h[5], w18 +; CHECK-GI-NEXT: sxtb w18, w3 +; CHECK-GI-NEXT: sxtb w3, w24 +; CHECK-GI-NEXT: mov v23.h[1], w26 +; CHECK-GI-NEXT: mov v21.h[2], w21 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: fmov w28, s24 +; CHECK-GI-NEXT: mov v22.h[3], w7 +; CHECK-GI-NEXT: sxtb w7, w20 +; CHECK-GI-NEXT: mov v19.h[2], w4 +; CHECK-GI-NEXT: sxtb w4, w30 +; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v18.h[3], w15 +; CHECK-GI-NEXT: sxtb w20, w28 +; CHECK-GI-NEXT: sxtb w15, w17 +; CHECK-GI-NEXT: sxtb w17, w22 +; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov v23.h[2], w25 +; CHECK-GI-NEXT: mov v20.h[6], w3 +; CHECK-GI-NEXT: mov v21.h[3], w7 +; CHECK-GI-NEXT: fmov w10, s26 +; CHECK-GI-NEXT: mov v22.h[4], w4 +; CHECK-GI-NEXT: mov v19.h[3], w18 +; CHECK-GI-NEXT: sxtb w18, w23 +; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload +; CHECK-GI-NEXT: mov b27, v17.b[13] +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v23.h[3], w20 +; CHECK-GI-NEXT: mov v18.h[4], w13 +; CHECK-GI-NEXT: sxtb w13, w6 +; CHECK-GI-NEXT: mov v20.h[7], w17 +; CHECK-GI-NEXT: mov v21.h[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #8] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v22.h[5], w18 +; CHECK-GI-NEXT: mov b25, v17.b[14] +; CHECK-GI-NEXT: fmov w26, s27 +; CHECK-GI-NEXT: mov v19.h[4], w15 +; CHECK-GI-NEXT: fmov w14, s29 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v23.h[4], w10 +; CHECK-GI-NEXT: sxtb w10, w11 +; CHECK-GI-NEXT: sxtb w11, w16 +; CHECK-GI-NEXT: mov v21.h[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #4] // 4-byte Folded Reload +; CHECK-GI-NEXT: sxtb w15, w26 +; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov v18.h[5], w10 +; CHECK-GI-NEXT: sxtb w10, w19 +; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload +; CHECK-GI-NEXT: mul v20.8h, v7.8h, v20.8h +; CHECK-GI-NEXT: mov b7, v17.b[7] +; CHECK-GI-NEXT: mov v22.h[6], w13 +; CHECK-GI-NEXT: sxtb w13, w5 +; CHECK-GI-NEXT: fmov w27, s25 +; CHECK-GI-NEXT: mov v19.h[5], w11 +; CHECK-GI-NEXT: sxtb w11, w2 +; CHECK-GI-NEXT: mov b17, v17.b[15] +; CHECK-GI-NEXT: mov v18.h[6], w8 +; CHECK-GI-NEXT: mov v16.h[7], w9 +; CHECK-GI-NEXT: sxtb w9, w14 +; CHECK-GI-NEXT: mov v23.h[5], w15 +; CHECK-GI-NEXT: mov v21.h[6], w10 +; CHECK-GI-NEXT: sxtb w14, w27 +; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: fmov w8, s7 +; CHECK-GI-NEXT: mov v22.h[7], w13 +; CHECK-GI-NEXT: fmov w12, s30 +; CHECK-GI-NEXT: mov v19.h[6], w9 +; CHECK-GI-NEXT: fmov w9, s17 +; CHECK-GI-NEXT: smov w10, v20.h[0] +; CHECK-GI-NEXT: mov v23.h[6], w14 +; CHECK-GI-NEXT: mov v18.h[7], w11 +; CHECK-GI-NEXT: smov w13, v20.h[1] +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: sxtb w12, w12 +; CHECK-GI-NEXT: smov w11, v20.h[4] +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mul v22.8h, v16.8h, v22.8h +; CHECK-GI-NEXT: smov w14, v20.h[3] +; CHECK-GI-NEXT: mov v21.h[7], w8 +; CHECK-GI-NEXT: ldrsb w8, [x0, #32] +; CHECK-GI-NEXT: mov v19.h[7], w12 +; CHECK-GI-NEXT: mov v23.h[7], w9 +; CHECK-GI-NEXT: ldrsb w9, [x1, #32] +; CHECK-GI-NEXT: fmov s7, w10 +; CHECK-GI-NEXT: smov w10, v20.h[2] +; CHECK-GI-NEXT: smov w12, v20.h[5] +; CHECK-GI-NEXT: fmov s16, w11 +; CHECK-GI-NEXT: mul w9, w9, w8 +; CHECK-GI-NEXT: smov w15, v22.h[4] +; CHECK-GI-NEXT: smov w17, v22.h[5] +; CHECK-GI-NEXT: mul v24.8h, v18.8h, v21.8h +; CHECK-GI-NEXT: mov v7.s[1], w13 +; CHECK-GI-NEXT: smov w13, v22.h[0] +; CHECK-GI-NEXT: mul v18.8h, v19.8h, v23.8h +; CHECK-GI-NEXT: smov w16, v22.h[1] +; CHECK-GI-NEXT: smov w8, v20.h[7] +; CHECK-GI-NEXT: sxth w9, w9 +; CHECK-GI-NEXT: mov v16.s[1], w12 +; CHECK-GI-NEXT: fmov s0, wzr +; CHECK-GI-NEXT: fmov s19, w15 +; CHECK-GI-NEXT: smov w15, v22.h[6] ; CHECK-GI-NEXT: mov v1.s[1], wzr +; CHECK-GI-NEXT: smov w11, v24.h[0] +; CHECK-GI-NEXT: mov v7.s[2], w10 +; CHECK-GI-NEXT: smov w10, v20.h[6] +; CHECK-GI-NEXT: smov w12, v24.h[1] +; CHECK-GI-NEXT: smov w0, v18.h[4] +; CHECK-GI-NEXT: fmov s17, w13 +; CHECK-GI-NEXT: mov v19.s[1], w17 +; CHECK-GI-NEXT: smov w17, v18.h[0] +; CHECK-GI-NEXT: smov w18, v18.h[1] +; CHECK-GI-NEXT: smov w13, v22.h[2] ; CHECK-GI-NEXT: mov v3.s[1], wzr ; CHECK-GI-NEXT: mov v2.s[1], wzr -; CHECK-GI-NEXT: fmov s20, w15 -; CHECK-GI-NEXT: smov w15, v25.b[13] +; CHECK-GI-NEXT: fmov s20, w11 +; CHECK-GI-NEXT: smov w11, v24.h[4] +; CHECK-GI-NEXT: mov v7.s[3], w14 +; CHECK-GI-NEXT: smov w14, v24.h[5] +; CHECK-GI-NEXT: mov v17.s[1], w16 +; CHECK-GI-NEXT: smov w16, v24.h[2] +; CHECK-GI-NEXT: mov v19.s[2], w15 +; CHECK-GI-NEXT: smov w15, v18.h[5] +; CHECK-GI-NEXT: fmov s23, w0 +; CHECK-GI-NEXT: mov v20.s[1], w12 +; CHECK-GI-NEXT: mov v16.s[2], w10 +; CHECK-GI-NEXT: smov w10, v22.h[3] +; CHECK-GI-NEXT: fmov s21, w11 +; CHECK-GI-NEXT: smov w11, v22.h[7] +; CHECK-GI-NEXT: fmov s22, w17 +; CHECK-GI-NEXT: mov v5.s[1], wzr ; CHECK-GI-NEXT: mov v4.s[1], wzr -; CHECK-GI-NEXT: fmov s19, w10 -; CHECK-GI-NEXT: mov v7.s[2], w12 -; CHECK-GI-NEXT: smov w12, v21.b[10] -; CHECK-GI-NEXT: mov v18.s[1], w14 -; CHECK-GI-NEXT: smov w14, v25.b[5] -; CHECK-GI-NEXT: mov v17.s[2], w8 -; CHECK-GI-NEXT: smov w8, v21.b[11] -; CHECK-GI-NEXT: smov w10, v21.b[14] -; CHECK-GI-NEXT: mov v5.s[2], wzr -; CHECK-GI-NEXT: mov v19.s[1], w11 -; CHECK-GI-NEXT: smov w11, v25.b[2] -; CHECK-GI-NEXT: mov v6.s[2], wzr -; CHECK-GI-NEXT: mov v16.s[2], w12 -; CHECK-GI-NEXT: smov w12, v25.b[8] -; CHECK-GI-NEXT: mov v7.s[3], w9 -; CHECK-GI-NEXT: mov v20.s[1], w14 -; CHECK-GI-NEXT: smov w14, v21.b[15] -; CHECK-GI-NEXT: smov w9, v25.b[9] -; CHECK-GI-NEXT: mov v17.s[3], w13 -; CHECK-GI-NEXT: smov w13, v25.b[12] -; CHECK-GI-NEXT: mov v0.s[2], wzr -; CHECK-GI-NEXT: mov v18.s[2], w11 -; CHECK-GI-NEXT: smov w11, v26.b[0] -; CHECK-GI-NEXT: mov v19.s[2], w10 -; CHECK-GI-NEXT: fmov s21, w12 -; CHECK-GI-NEXT: smov w12, v26.b[1] -; CHECK-GI-NEXT: mov v16.s[3], w8 -; CHECK-GI-NEXT: smov w8, v26.b[5] -; CHECK-GI-NEXT: smov w10, v25.b[6] +; CHECK-GI-NEXT: mov v6.s[1], wzr +; CHECK-GI-NEXT: mov v23.s[1], w15 +; CHECK-GI-NEXT: smov w15, v18.h[6] +; CHECK-GI-NEXT: mov v0.s[1], wzr +; CHECK-GI-NEXT: mov v21.s[1], w14 +; CHECK-GI-NEXT: smov w14, v24.h[6] +; CHECK-GI-NEXT: mov v20.s[2], w16 +; CHECK-GI-NEXT: mov v22.s[1], w18 +; CHECK-GI-NEXT: smov w16, v18.h[2] ; CHECK-GI-NEXT: mov v1.s[2], wzr -; CHECK-GI-NEXT: fmov s23, w13 -; CHECK-GI-NEXT: smov w13, v25.b[3] ; CHECK-GI-NEXT: mov v3.s[2], wzr -; CHECK-GI-NEXT: fmov s24, w11 -; CHECK-GI-NEXT: mov v21.s[1], w9 -; CHECK-GI-NEXT: smov w9, v25.b[10] -; CHECK-GI-NEXT: smov w11, v26.b[2] -; CHECK-GI-NEXT: mov v19.s[3], w14 -; CHECK-GI-NEXT: smov w14, v26.b[13] -; CHECK-GI-NEXT: mov v23.s[1], w15 -; CHECK-GI-NEXT: smov w15, v25.b[14] -; CHECK-GI-NEXT: mov v20.s[2], w10 -; CHECK-GI-NEXT: mov v24.s[1], w12 -; CHECK-GI-NEXT: smov w12, v26.b[4] -; CHECK-GI-NEXT: smov w10, v25.b[7] -; CHECK-GI-NEXT: mov v21.s[2], w9 -; CHECK-GI-NEXT: smov w9, v25.b[11] -; CHECK-GI-NEXT: mov v18.s[3], w13 -; CHECK-GI-NEXT: smov w13, v26.b[9] ; CHECK-GI-NEXT: mov v2.s[2], wzr +; CHECK-GI-NEXT: mov v5.s[2], wzr ; CHECK-GI-NEXT: mov v4.s[2], wzr +; CHECK-GI-NEXT: mov v6.s[2], wzr ; CHECK-GI-NEXT: mov v23.s[2], w15 -; CHECK-GI-NEXT: smov w15, v25.b[15] -; CHECK-GI-NEXT: mov v5.s[3], wzr -; CHECK-GI-NEXT: fmov s27, w12 -; CHECK-GI-NEXT: mov v24.s[2], w11 -; CHECK-GI-NEXT: smov w11, v26.b[6] -; CHECK-GI-NEXT: smov w12, v26.b[8] -; CHECK-GI-NEXT: mov v21.s[3], w9 -; CHECK-GI-NEXT: smov w9, v26.b[12] -; CHECK-GI-NEXT: mov v20.s[3], w10 -; CHECK-GI-NEXT: smov w10, v26.b[3] -; CHECK-GI-NEXT: mov v6.s[3], wzr -; CHECK-GI-NEXT: mov v27.s[1], w8 -; CHECK-GI-NEXT: mov v23.s[3], w15 -; CHECK-GI-NEXT: smov w15, v22.b[0] -; CHECK-GI-NEXT: smov w8, v26.b[7] -; CHECK-GI-NEXT: mov v0.s[3], wzr +; CHECK-GI-NEXT: mov v21.s[2], w14 +; CHECK-GI-NEXT: smov w14, v18.h[3] +; CHECK-GI-NEXT: smov w15, v18.h[7] +; CHECK-GI-NEXT: fmov s18, w9 +; CHECK-GI-NEXT: ldr w9, [sp, #12] // 4-byte Folded Reload +; CHECK-GI-NEXT: mov v17.s[2], w13 +; CHECK-GI-NEXT: smov w12, v24.h[3] +; CHECK-GI-NEXT: smov w13, v24.h[7] +; CHECK-GI-NEXT: mov v22.s[2], w16 +; CHECK-GI-NEXT: mov v0.s[2], wzr ; CHECK-GI-NEXT: mov v1.s[3], wzr -; CHECK-GI-NEXT: fmov s25, w12 -; CHECK-GI-NEXT: fmov s29, w9 -; CHECK-GI-NEXT: smov w9, v22.b[5] -; CHECK-GI-NEXT: mov v24.s[3], w10 -; CHECK-GI-NEXT: smov w10, v22.b[1] -; CHECK-GI-NEXT: smov w12, v26.b[10] -; CHECK-GI-NEXT: mov v27.s[2], w11 -; CHECK-GI-NEXT: smov w11, v22.b[4] -; CHECK-GI-NEXT: fmov s28, w15 -; CHECK-GI-NEXT: mov v25.s[1], w13 -; CHECK-GI-NEXT: smov w13, v26.b[14] -; CHECK-GI-NEXT: mov v29.s[1], w14 -; CHECK-GI-NEXT: smov w15, v22.b[12] -; CHECK-GI-NEXT: smov w14, v22.b[2] ; CHECK-GI-NEXT: mov v3.s[3], wzr -; CHECK-GI-NEXT: mov v28.s[1], w10 -; CHECK-GI-NEXT: smov w10, v22.b[13] ; CHECK-GI-NEXT: mov v2.s[3], wzr -; CHECK-GI-NEXT: fmov s30, w11 -; CHECK-GI-NEXT: smov w11, v22.b[6] -; CHECK-GI-NEXT: mov v27.s[3], w8 -; CHECK-GI-NEXT: mov v25.s[2], w12 -; CHECK-GI-NEXT: mov v29.s[2], w13 -; CHECK-GI-NEXT: smov w13, v26.b[11] -; CHECK-GI-NEXT: fmov s31, w15 -; CHECK-GI-NEXT: smov w15, v26.b[15] -; CHECK-GI-NEXT: smov w12, v22.b[9] -; CHECK-GI-NEXT: mov v30.s[1], w9 -; CHECK-GI-NEXT: smov w9, v22.b[8] -; CHECK-GI-NEXT: mov v28.s[2], w14 -; CHECK-GI-NEXT: ldrsb w14, [x1, #32] -; CHECK-GI-NEXT: smov w8, v22.b[15] -; CHECK-GI-NEXT: mul v17.4s, v17.4s, v27.4s -; CHECK-GI-NEXT: mov v31.s[1], w10 -; CHECK-GI-NEXT: smov w10, v22.b[14] -; CHECK-GI-NEXT: mov v25.s[3], w13 -; CHECK-GI-NEXT: ldrsb w13, [x0, #32] -; CHECK-GI-NEXT: mov v29.s[3], w15 +; CHECK-GI-NEXT: mov v5.s[3], wzr ; CHECK-GI-NEXT: mov v4.s[3], wzr -; CHECK-GI-NEXT: mov v30.s[2], w11 -; CHECK-GI-NEXT: fmov s26, w9 -; CHECK-GI-NEXT: smov w9, v22.b[7] -; CHECK-GI-NEXT: smov w11, v22.b[3] -; CHECK-GI-NEXT: add v5.4s, v5.4s, v6.4s -; CHECK-GI-NEXT: mla v17.4s, v7.4s, v24.4s -; CHECK-GI-NEXT: mov v31.s[2], w10 +; CHECK-GI-NEXT: mov v6.s[3], wzr +; CHECK-GI-NEXT: mov v18.s[1], wzr +; CHECK-GI-NEXT: mov v16.s[3], w8 +; CHECK-GI-NEXT: mov v17.s[3], w10 +; CHECK-GI-NEXT: mov v19.s[3], w11 +; CHECK-GI-NEXT: mov v20.s[3], w12 +; CHECK-GI-NEXT: mov v21.s[3], w13 +; CHECK-GI-NEXT: mov v22.s[3], w14 +; CHECK-GI-NEXT: mov v23.s[3], w15 +; CHECK-GI-NEXT: mov v0.s[3], wzr ; CHECK-GI-NEXT: add v1.4s, v1.4s, v3.4s -; CHECK-GI-NEXT: mov v26.s[1], w12 -; CHECK-GI-NEXT: smov w12, v22.b[10] -; CHECK-GI-NEXT: mul v19.4s, v19.4s, v29.4s -; CHECK-GI-NEXT: mov v30.s[3], w9 -; CHECK-GI-NEXT: mul w9, w14, w13 -; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s -; CHECK-GI-NEXT: mov v28.s[3], w11 -; CHECK-GI-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-GI-NEXT: mov v31.s[3], w8 -; CHECK-GI-NEXT: smov w8, v22.b[11] -; CHECK-GI-NEXT: fmov s8, w9 -; CHECK-GI-NEXT: mov v26.s[2], w12 -; CHECK-GI-NEXT: mla v19.4s, v16.4s, v25.4s -; CHECK-GI-NEXT: mul v20.4s, v20.4s, v30.4s -; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s -; CHECK-GI-NEXT: mov v8.s[1], wzr -; CHECK-GI-NEXT: mul v22.4s, v23.4s, v31.4s -; CHECK-GI-NEXT: mov v26.s[3], w8 -; CHECK-GI-NEXT: add v3.4s, v17.4s, v19.4s -; CHECK-GI-NEXT: mla v20.4s, v18.4s, v28.4s -; CHECK-GI-NEXT: mov v8.s[2], wzr -; CHECK-GI-NEXT: mla v22.4s, v21.4s, v26.4s -; CHECK-GI-NEXT: mov v8.s[3], wzr -; CHECK-GI-NEXT: add v4.4s, v20.4s, v22.4s -; CHECK-GI-NEXT: add v0.4s, v8.4s, v0.4s -; CHECK-GI-NEXT: add v2.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: add v2.4s, v2.4s, v5.4s +; CHECK-GI-NEXT: add v3.4s, v4.4s, v6.4s +; CHECK-GI-NEXT: mov v18.s[2], wzr +; CHECK-GI-NEXT: add v4.4s, v7.4s, v16.4s +; CHECK-GI-NEXT: add v5.4s, v17.4s, v19.4s +; CHECK-GI-NEXT: add v6.4s, v20.4s, v21.4s +; CHECK-GI-NEXT: add v7.4s, v22.4s, v23.4s ; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-NEXT: add v0.4s, v2.4s, v0.4s +; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: mov v18.s[3], wzr +; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: add v3.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v18.4s, v0.4s +; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ldr d8, [sp], #16 // 8-byte Folded Reload +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: add sp, sp, #112 ; CHECK-GI-NEXT: ret entry: %0 = load <33 x i8>, ptr %a @@ -4845,13 +5844,12 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 ; ; CHECK-GI-LABEL: test_sdot_v33i8_double: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: sub sp, sp, #96 -; CHECK-GI-NEXT: stp d15, d14, [sp, #16] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp d13, d12, [sp, #32] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp d11, d10, [sp, #48] // 16-byte Folded Spill -; CHECK-GI-NEXT: stp d9, d8, [sp, #64] // 16-byte Folded Spill -; CHECK-GI-NEXT: str x29, [sp, #80] // 8-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 96 +; CHECK-GI-NEXT: stp d15, d14, [sp, #-80]! // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: str x29, [sp, #64] // 8-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 80 ; CHECK-GI-NEXT: .cfi_offset w29, -16 ; CHECK-GI-NEXT: .cfi_offset b8, -24 ; CHECK-GI-NEXT: .cfi_offset b9, -32 @@ -4861,508 +5859,762 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 ; CHECK-GI-NEXT: .cfi_offset b13, -64 ; CHECK-GI-NEXT: .cfi_offset b14, -72 ; CHECK-GI-NEXT: .cfi_offset b15, -80 -; CHECK-GI-NEXT: sxtb w8, w0 -; CHECK-GI-NEXT: sxtb w9, w1 -; CHECK-GI-NEXT: sxtb w10, w2 -; CHECK-GI-NEXT: sxtb w11, w4 -; CHECK-GI-NEXT: sxtb w12, w5 -; CHECK-GI-NEXT: sxtb w13, w7 -; CHECK-GI-NEXT: fmov s28, w8 +; CHECK-GI-NEXT: lsl w8, w0, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #80] +; CHECK-GI-NEXT: lsl w11, w1, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #88] +; CHECK-GI-NEXT: ldr w13, [sp, #128] +; CHECK-GI-NEXT: ldr w14, [sp, #136] +; CHECK-GI-NEXT: sbfx w12, w8, #8, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sbfx w8, w11, #8, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: lsl w11, w2, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: fmov s22, w12 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: ldr w12, [sp, #152] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: lsl w16, w7, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: fmov s23, w10 +; CHECK-GI-NEXT: sbfx w10, w11, #8, #8 +; CHECK-GI-NEXT: lsl w11, w3, #8 +; CHECK-GI-NEXT: mov v22.h[1], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #96] -; CHECK-GI-NEXT: fmov s0, wzr -; CHECK-GI-NEXT: fmov s25, w11 -; CHECK-GI-NEXT: sxtb w11, w6 -; CHECK-GI-NEXT: ldr w14, [sp, #528] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: fmov s18, wzr -; CHECK-GI-NEXT: fmov s20, wzr -; CHECK-GI-NEXT: mov v28.s[1], w9 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: ldr w15, [sp, #176] +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: mov v23.h[1], w9 ; CHECK-GI-NEXT: ldr w9, [sp, #104] -; CHECK-GI-NEXT: str q0, [sp] // 16-byte Folded Spill -; CHECK-GI-NEXT: fmov s24, w8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: ldr w17, [sp, #224] +; CHECK-GI-NEXT: mov v22.h[2], w10 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w11, #8, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: lsl w11, w4, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: mov v23.h[2], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #112] -; CHECK-GI-NEXT: mov v25.s[1], w12 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w12, [sp, #136] -; CHECK-GI-NEXT: mov v18.s[1], wzr -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v20.s[1], wzr ; CHECK-GI-NEXT: fmov s19, wzr -; CHECK-GI-NEXT: mov v28.s[2], w10 -; CHECK-GI-NEXT: sxtb w10, w3 -; CHECK-GI-NEXT: mov v24.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #128] -; CHECK-GI-NEXT: mov v25.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #168] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v18.s[2], wzr +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 ; CHECK-GI-NEXT: fmov s21, wzr -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v20.s[2], wzr -; CHECK-GI-NEXT: mov v28.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #160] -; CHECK-GI-NEXT: mov v24.s[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #120] -; CHECK-GI-NEXT: fmov s30, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #144] -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v25.s[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #200] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v19.s[1], wzr -; CHECK-GI-NEXT: fmov s22, w10 -; CHECK-GI-NEXT: mov v30.s[1], w12 -; CHECK-GI-NEXT: ldr w10, [sp, #176] -; CHECK-GI-NEXT: mov v24.s[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #224] -; CHECK-GI-NEXT: ldr w12, [sp, #152] -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: mov v21.s[1], wzr -; CHECK-GI-NEXT: mov v22.s[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #192] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v30.s[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #232] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: fmov s23, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #240] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v18.s[3], wzr -; CHECK-GI-NEXT: mov v20.s[3], wzr -; CHECK-GI-NEXT: mov v22.s[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #184] -; CHECK-GI-NEXT: fmov s26, w11 -; CHECK-GI-NEXT: mov v23.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #256] -; CHECK-GI-NEXT: ldr w11, [sp, #208] -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v30.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #264] -; CHECK-GI-NEXT: mov v26.s[1], w13 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v22.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #296] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: fmov s29, w9 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w13, [sp, #216] -; CHECK-GI-NEXT: sxtb w9, w10 -; CHECK-GI-NEXT: mov v23.s[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #248] -; CHECK-GI-NEXT: mov v26.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #304] -; CHECK-GI-NEXT: ldr w10, [sp, #272] -; CHECK-GI-NEXT: fmov s31, w9 -; CHECK-GI-NEXT: mov v29.s[1], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #312] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: ldr w12, [sp, #280] +; CHECK-GI-NEXT: mov v22.h[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #144] +; CHECK-GI-NEXT: lsl w8, w8, #8 ; CHECK-GI-NEXT: fmov s16, wzr -; CHECK-GI-NEXT: mov v31.s[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #328] -; CHECK-GI-NEXT: mov v23.s[3], w8 -; CHECK-GI-NEXT: sxtb w8, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #360] -; CHECK-GI-NEXT: mov v29.s[2], w10 -; CHECK-GI-NEXT: sxtb w10, w11 -; CHECK-GI-NEXT: mov v26.s[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #336] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w11, [sp, #368] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v31.s[2], w8 -; CHECK-GI-NEXT: fmov s0, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #320] -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: fmov s12, w9 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v29.s[3], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #376] -; CHECK-GI-NEXT: mov v0.s[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #344] -; CHECK-GI-NEXT: ldr w8, [sp, #288] -; CHECK-GI-NEXT: mov v12.s[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #392] -; CHECK-GI-NEXT: mov v31.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #424] -; CHECK-GI-NEXT: sxtb w12, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #400] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v0.s[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #432] -; CHECK-GI-NEXT: fmov s13, w11 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: mov v12.s[2], w9 -; CHECK-GI-NEXT: fmov s8, w10 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w10, [sp, #440] -; CHECK-GI-NEXT: ldr w11, [sp, #384] -; CHECK-GI-NEXT: ldr w9, [sp, #352] +; CHECK-GI-NEXT: fmov s18, wzr ; CHECK-GI-NEXT: fmov s17, wzr -; CHECK-GI-NEXT: mov v13.s[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #408] -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v8.s[1], w12 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: ldr w12, [sp, #456] -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: fmov s3, wzr -; CHECK-GI-NEXT: mov v12.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #488] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v13.s[2], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #496] -; CHECK-GI-NEXT: mov v0.s[3], w9 -; CHECK-GI-NEXT: mov v8.s[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #416] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: ldr w9, [sp, #464] -; CHECK-GI-NEXT: fmov s14, w12 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: fmov s9, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #504] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w12, [sp, #448] -; CHECK-GI-NEXT: mul v27.4s, v25.4s, v0.4s -; CHECK-GI-NEXT: mov v13.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #560] -; CHECK-GI-NEXT: sxtb w15, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #568] -; CHECK-GI-NEXT: mov v9.s[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #520] -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v14.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #472] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: fmov s10, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #552] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: fmov s15, w13 -; CHECK-GI-NEXT: mov v8.s[3], w12 -; CHECK-GI-NEXT: sxtb w12, w14 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: mov v14.s[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #480] -; CHECK-GI-NEXT: mov v10.s[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #576] -; CHECK-GI-NEXT: mov v9.s[2], w15 -; CHECK-GI-NEXT: mul w8, w8, w10 -; CHECK-GI-NEXT: mov v15.s[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #512] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w10, [sp, #584] -; CHECK-GI-NEXT: ldr w13, [sp, #536] -; CHECK-GI-NEXT: mla v27.4s, v28.4s, v31.4s -; CHECK-GI-NEXT: mul v30.4s, v30.4s, v13.4s -; CHECK-GI-NEXT: mov v10.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #592] -; CHECK-GI-NEXT: fmov s25, w8 -; CHECK-GI-NEXT: mov v14.s[3], w9 -; CHECK-GI-NEXT: sxtb w9, w12 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w8, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #624] -; CHECK-GI-NEXT: sxtb w13, w13 -; CHECK-GI-NEXT: mov v9.s[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #600] -; CHECK-GI-NEXT: mla v30.4s, v24.4s, v12.4s -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v10.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #632] -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #656] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: fmov s28, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #688] -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v15.s[2], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #544] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v0.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #664] -; CHECK-GI-NEXT: mov v28.s[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #696] -; CHECK-GI-NEXT: fmov s11, w8 -; CHECK-GI-NEXT: fmov s31, w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w12, w13 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: ldr w11, [sp, #672] -; CHECK-GI-NEXT: ldr w8, [sp, #616] -; CHECK-GI-NEXT: mov v11.s[1], w9 -; CHECK-GI-NEXT: mov v15.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #608] -; CHECK-GI-NEXT: mov v31.s[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #704] -; CHECK-GI-NEXT: ldr w9, [sp, #640] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mul v24.4s, v26.4s, v14.4s -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v11.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #712] -; CHECK-GI-NEXT: mov v0.s[2], w12 -; CHECK-GI-NEXT: mov v31.s[2], w10 -; CHECK-GI-NEXT: ldr w12, [sp, #648] -; CHECK-GI-NEXT: mov v28.s[2], w9 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: ldr w10, [sp, #720] -; CHECK-GI-NEXT: ldr w9, [sp, #680] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mul v26.4s, v29.4s, v15.4s -; CHECK-GI-NEXT: mla v24.4s, v22.4s, v8.4s -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v0.s[3], w8 -; CHECK-GI-NEXT: mov v31.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #784] -; CHECK-GI-NEXT: mov v28.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #752] -; CHECK-GI-NEXT: fmov s13, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #792] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mov v11.s[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #760] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: ldr w8, [sp, #728] -; CHECK-GI-NEXT: fmov s14, w11 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w11, [sp, #744] -; CHECK-GI-NEXT: fmov s12, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #824] -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: mla v26.4s, v23.4s, v9.4s -; CHECK-GI-NEXT: ldr w13, [sp, #984] -; CHECK-GI-NEXT: mov v14.s[1], w10 -; CHECK-GI-NEXT: sxtb w10, w12 -; CHECK-GI-NEXT: mov v13.s[1], w8 -; CHECK-GI-NEXT: mov v12.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #832] -; CHECK-GI-NEXT: ldr w8, [sp, #736] -; CHECK-GI-NEXT: fmov s29, w10 -; CHECK-GI-NEXT: ldr w12, [sp, #768] -; CHECK-GI-NEXT: ldr w10, [sp, #800] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v23.h[3], w9 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #120] +; CHECK-GI-NEXT: fmov s20, wzr ; CHECK-GI-NEXT: fmov s6, wzr -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: mov v22.h[4], w11 +; CHECK-GI-NEXT: lsl w11, w5, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: fmov s7, wzr ; CHECK-GI-NEXT: fmov s2, wzr -; CHECK-GI-NEXT: mov v29.s[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #840] -; CHECK-GI-NEXT: mov v13.s[2], w8 -; CHECK-GI-NEXT: mov v12.s[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #808] -; CHECK-GI-NEXT: mov v14.s[2], w10 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: ldr w8, [sp, #776] -; CHECK-GI-NEXT: ldr w10, [sp, #848] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: fmov s5, wzr +; CHECK-GI-NEXT: fmov s24, w10 +; CHECK-GI-NEXT: mov v23.h[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #160] +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #168] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: lsl w8, w8, #8 ; CHECK-GI-NEXT: fmov s4, wzr -; CHECK-GI-NEXT: mov v29.s[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #856] -; CHECK-GI-NEXT: mov v13.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #864] -; CHECK-GI-NEXT: mov v14.s[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #888] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: fmov s7, wzr -; CHECK-GI-NEXT: fmov s15, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #920] -; CHECK-GI-NEXT: mov v12.s[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #872] -; CHECK-GI-NEXT: mov v29.s[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #896] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: fmov s22, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #928] -; CHECK-GI-NEXT: mov v15.s[1], w11 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: fmov s8, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #952] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: ldr w11, [sp, #904] -; CHECK-GI-NEXT: mov v22.s[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #936] -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: mov v19.s[2], wzr -; CHECK-GI-NEXT: mov v21.s[2], wzr -; CHECK-GI-NEXT: mov v15.s[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #960] -; CHECK-GI-NEXT: mov v8.s[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #880] -; CHECK-GI-NEXT: fmov s23, w9 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: ldr w9, [sp, #944] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v22.s[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #912] -; CHECK-GI-NEXT: mov v8.s[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #968] -; CHECK-GI-NEXT: mov v23.s[1], w8 -; CHECK-GI-NEXT: mov v15.s[3], w12 -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w12, w13 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: add v18.4s, v18.4s, v20.4s -; CHECK-GI-NEXT: mov v22.s[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #992] -; CHECK-GI-NEXT: fmov s9, w12 -; CHECK-GI-NEXT: mov v23.s[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1048] -; CHECK-GI-NEXT: ldr w12, [sp, #1056] -; CHECK-GI-NEXT: mul v0.4s, v0.4s, v15.4s -; CHECK-GI-NEXT: sxtb w13, w11 -; CHECK-GI-NEXT: mov v8.s[3], w9 -; CHECK-GI-NEXT: sxtb w11, w10 -; CHECK-GI-NEXT: ldr w9, [sp, #1000] -; CHECK-GI-NEXT: sxtb w12, w12 -; CHECK-GI-NEXT: mov v9.s[1], w13 -; CHECK-GI-NEXT: ldr w10, [sp, #1016] -; CHECK-GI-NEXT: ldr w8, [sp, #816] -; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: fmov s3, wzr +; CHECK-GI-NEXT: mov v24.h[1], w12 +; CHECK-GI-NEXT: lsl w12, w6, #8 +; CHECK-GI-NEXT: mov v22.h[5], w11 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v23.h[5], w9 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #184] +; CHECK-GI-NEXT: ldr w9, [sp, #192] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: fmov s5, wzr ; CHECK-GI-NEXT: fmov s1, wzr +; CHECK-GI-NEXT: mov v24.h[2], w8 +; CHECK-GI-NEXT: mov v22.h[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #208] +; CHECK-GI-NEXT: mov v23.h[6], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #216] +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: ldr w8, [sp, #200] +; CHECK-GI-NEXT: fmov s0, wzr +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v19.s[1], wzr +; CHECK-GI-NEXT: mov v24.h[3], w10 +; CHECK-GI-NEXT: sbfx w10, w14, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #280] +; CHECK-GI-NEXT: mov v22.h[7], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #288] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v23.h[7], w10 +; CHECK-GI-NEXT: lsl w18, w16, #8 +; CHECK-GI-NEXT: fmov s27, w12 +; CHECK-GI-NEXT: ldr w10, [sp, #232] +; CHECK-GI-NEXT: sbfx w16, w14, #8, #8 +; CHECK-GI-NEXT: mov v24.h[4], w15 +; CHECK-GI-NEXT: lsl w15, w11, #8 +; CHECK-GI-NEXT: sbfx w14, w18, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #296] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: fmov s25, w16 +; CHECK-GI-NEXT: ldr w16, [sp, #344] +; CHECK-GI-NEXT: mov v27.h[1], w13 +; CHECK-GI-NEXT: lsl w13, w17, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: ldr w12, [sp, #240] +; CHECK-GI-NEXT: sbfx w17, w10, #8, #8 +; CHECK-GI-NEXT: mov v25.h[1], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #352] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v24.h[5], w15 +; CHECK-GI-NEXT: mov v27.h[2], w13 +; CHECK-GI-NEXT: lsl w13, w14, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #304] +; CHECK-GI-NEXT: fmov s26, w16 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: ldr w15, [sp, #248] +; CHECK-GI-NEXT: mov v25.h[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #360] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: mov v24.h[6], w9 +; CHECK-GI-NEXT: lsl w16, w11, #8 +; CHECK-GI-NEXT: mov v26.h[1], w13 +; CHECK-GI-NEXT: mov v27.h[3], w17 +; CHECK-GI-NEXT: sbfx w13, w14, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #312] +; CHECK-GI-NEXT: ldr w17, [sp, #328] +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #256] +; CHECK-GI-NEXT: ldr w11, [sp, #264] +; CHECK-GI-NEXT: mov v25.h[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #368] +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: mov v26.h[2], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #320] +; CHECK-GI-NEXT: mov v27.h[4], w12 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w9, w14, #8, #8 +; CHECK-GI-NEXT: lsl w14, w15, #8 +; CHECK-GI-NEXT: lsl w15, w16, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #408] +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sbfx w12, w13, #8, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #376] +; CHECK-GI-NEXT: mov v25.h[4], w9 +; CHECK-GI-NEXT: sbfx w9, w14, #8, #8 +; CHECK-GI-NEXT: sbfx w14, w15, #8, #8 +; CHECK-GI-NEXT: lsl w15, w16, #8 +; CHECK-GI-NEXT: mov v26.h[3], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #416] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: lsl w16, w17, #8 +; CHECK-GI-NEXT: mov v27.h[5], w9 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v25.h[5], w14 +; CHECK-GI-NEXT: fmov s29, w15 +; CHECK-GI-NEXT: ldr w14, [sp, #384] +; CHECK-GI-NEXT: ldr w15, [sp, #472] +; CHECK-GI-NEXT: mov v26.h[4], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #424] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v29.h[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #480] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v25.h[6], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #432] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v26.h[5], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #392] +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: mov v29.h[2], w13 +; CHECK-GI-NEXT: fmov s28, w15 +; CHECK-GI-NEXT: ldr w9, [sp, #336] +; CHECK-GI-NEXT: ldr w13, [sp, #488] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: ldr w15, [sp, #440] +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v28.h[1], w12 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: mov v29.h[3], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #496] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: ldr w12, [sp, #400] +; CHECK-GI-NEXT: mov v26.h[6], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #448] +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: mov v28.h[2], w13 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: mov v25.h[7], w9 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v29.h[4], w15 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #456] +; CHECK-GI-NEXT: ldr w15, [sp, #504] +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: sbfx w9, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w12, w14, #8, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: lsl w14, w15, #8 +; CHECK-GI-NEXT: mov v28.h[3], w16 +; CHECK-GI-NEXT: ldr w15, [sp, #512] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mul v30.8h, v22.8h, v25.8h +; CHECK-GI-NEXT: mov v26.h[7], w9 +; CHECK-GI-NEXT: mov v29.h[5], w12 +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: sbfx w9, w14, #8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: sbfx w14, w11, #8, #8 +; CHECK-GI-NEXT: sbfx w11, w13, #8, #8 +; CHECK-GI-NEXT: lsl w13, w15, #8 +; CHECK-GI-NEXT: ldr w17, [sp, #464] +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: mov v28.h[4], w9 +; CHECK-GI-NEXT: mov v27.h[6], w10 +; CHECK-GI-NEXT: ldr w16, [sp, #520] +; CHECK-GI-NEXT: sbfx w10, w13, #8, #8 +; CHECK-GI-NEXT: smov w13, v30.h[0] +; CHECK-GI-NEXT: mov v24.h[7], w8 +; CHECK-GI-NEXT: lsl w8, w17, #8 +; CHECK-GI-NEXT: mov v29.h[6], w11 +; CHECK-GI-NEXT: mul v26.8h, v23.8h, v26.8h +; CHECK-GI-NEXT: lsl w15, w16, #8 +; CHECK-GI-NEXT: smov w16, v30.h[1] +; CHECK-GI-NEXT: ldr w12, [sp, #528] +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: mov v28.h[5], w10 +; CHECK-GI-NEXT: mov v27.h[7], w14 +; CHECK-GI-NEXT: fmov s22, w13 +; CHECK-GI-NEXT: sbfx w10, w15, #8, #8 +; CHECK-GI-NEXT: smov w14, v30.h[4] +; CHECK-GI-NEXT: mov v29.h[7], w8 +; CHECK-GI-NEXT: smov w15, v26.h[0] +; CHECK-GI-NEXT: smov w13, v30.h[2] +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #544] +; CHECK-GI-NEXT: ldr w11, [sp, #552] +; CHECK-GI-NEXT: mov v22.s[1], w16 +; CHECK-GI-NEXT: smov w16, v26.h[4] +; CHECK-GI-NEXT: mov v28.h[6], w10 +; CHECK-GI-NEXT: smov w10, v26.h[1] +; CHECK-GI-NEXT: fmov s23, w14 +; CHECK-GI-NEXT: smov w14, v26.h[5] +; CHECK-GI-NEXT: mul v29.8h, v24.8h, v29.8h +; CHECK-GI-NEXT: fmov s24, w15 +; CHECK-GI-NEXT: smov w15, v26.h[2] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: smov w8, v30.h[5] +; CHECK-GI-NEXT: smov w17, v30.h[7] +; CHECK-GI-NEXT: fmov s25, w16 +; CHECK-GI-NEXT: mov v22.s[2], w13 +; CHECK-GI-NEXT: smov w13, v30.h[3] +; CHECK-GI-NEXT: mov v24.s[1], w10 +; CHECK-GI-NEXT: smov w16, v26.h[6] +; CHECK-GI-NEXT: sbfx w10, w12, #8, #8 +; CHECK-GI-NEXT: smov w18, v29.h[0] +; CHECK-GI-NEXT: smov w0, v29.h[1] +; CHECK-GI-NEXT: ldr w12, [sp, #560] +; CHECK-GI-NEXT: mov v25.s[1], w14 +; CHECK-GI-NEXT: smov w14, v26.h[7] +; CHECK-GI-NEXT: mov v28.h[7], w10 +; CHECK-GI-NEXT: mov v22.s[3], w13 +; CHECK-GI-NEXT: smov w13, v26.h[3] +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v24.s[2], w15 +; CHECK-GI-NEXT: smov w15, v29.h[2] +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: fmov s26, w18 +; CHECK-GI-NEXT: mov v23.s[1], w8 +; CHECK-GI-NEXT: smov w8, v30.h[6] +; CHECK-GI-NEXT: mov v25.s[2], w16 +; CHECK-GI-NEXT: lsl w16, w9, #8 +; CHECK-GI-NEXT: mul v31.8h, v27.8h, v28.8h +; CHECK-GI-NEXT: ldr w10, [sp, #568] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #584] +; CHECK-GI-NEXT: mov v24.s[3], w13 +; CHECK-GI-NEXT: smov w13, v29.h[4] +; CHECK-GI-NEXT: mov v26.s[1], w0 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v23.s[2], w8 +; CHECK-GI-NEXT: mov v25.s[3], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #608] +; CHECK-GI-NEXT: ldr w8, [sp, #576] +; CHECK-GI-NEXT: fmov s8, w16 +; CHECK-GI-NEXT: ldr w16, [sp, #616] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: fmov s27, w13 +; CHECK-GI-NEXT: lsl w13, w14, #8 +; CHECK-GI-NEXT: mov v26.s[2], w15 +; CHECK-GI-NEXT: smov w15, v29.h[5] +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #624] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v8.h[1], w11 +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: mov v23.s[3], w17 +; CHECK-GI-NEXT: fmov s9, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #632] +; CHECK-GI-NEXT: smov w17, v31.h[1] +; CHECK-GI-NEXT: mov v27.s[1], w15 +; CHECK-GI-NEXT: smov w15, v31.h[0] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v8.h[2], w12 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: mov v9.h[1], w16 +; CHECK-GI-NEXT: smov w16, v31.h[2] +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #592] +; CHECK-GI-NEXT: ldr w12, [sp, #600] +; CHECK-GI-NEXT: fmov s28, w15 +; CHECK-GI-NEXT: smov w15, v29.h[6] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: mov v8.h[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #640] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v9.h[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #672] +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v28.s[1], w17 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: mov v27.s[2], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #680] +; CHECK-GI-NEXT: mov v8.h[4], w8 +; CHECK-GI-NEXT: smov w8, v31.h[4] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: mov v9.h[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #688] +; CHECK-GI-NEXT: mov v28.s[2], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #648] +; CHECK-GI-NEXT: fmov s10, w14 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #656] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: fmov s30, w8 +; CHECK-GI-NEXT: sbfx w8, w10, #8, #8 +; CHECK-GI-NEXT: smov w10, v31.h[5] +; CHECK-GI-NEXT: mov v8.h[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #696] +; CHECK-GI-NEXT: mov v10.h[1], w15 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v9.h[4], w8 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: ldr w8, [sp, #704] +; CHECK-GI-NEXT: ldr w15, [sp, #664] +; CHECK-GI-NEXT: ldr w17, [sp, #768] +; CHECK-GI-NEXT: mov v30.s[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #744] +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: mov v10.h[2], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #736] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v9.h[5], w16 +; CHECK-GI-NEXT: mov v8.h[6], w11 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #712] +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #720] +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v10.h[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #752] +; CHECK-GI-NEXT: mov v8.h[7], w12 +; CHECK-GI-NEXT: sbfx w12, w8, #8, #8 +; CHECK-GI-NEXT: lsl w18, w16, #8 +; CHECK-GI-NEXT: fmov s11, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #760] +; CHECK-GI-NEXT: ldr w8, [sp, #784] +; CHECK-GI-NEXT: mov v21.s[1], wzr ; CHECK-GI-NEXT: mov v16.s[1], wzr -; CHECK-GI-NEXT: mla v0.4s, v10.4s, v29.4s -; CHECK-GI-NEXT: fmov s10, w11 -; CHECK-GI-NEXT: sxtb w10, w10 -; CHECK-GI-NEXT: ldr w11, [sp, #1024] -; CHECK-GI-NEXT: mul v20.4s, v11.4s, v8.4s -; CHECK-GI-NEXT: ldr q8, [sp] // 16-byte Folded Reload -; CHECK-GI-NEXT: mov v9.s[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1008] -; CHECK-GI-NEXT: fmov s29, w10 -; CHECK-GI-NEXT: mov v10.s[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1064] -; CHECK-GI-NEXT: sxtb w11, w11 -; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov v18.s[1], wzr +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v10.h[4], w12 +; CHECK-GI-NEXT: sbfx w12, w15, #8, #8 +; CHECK-GI-NEXT: mov v11.h[1], w10 +; CHECK-GI-NEXT: sbfx w10, w14, #8, #8 +; CHECK-GI-NEXT: lsl w14, w9, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: ldr w9, [sp, #776] +; CHECK-GI-NEXT: lsl w8, w8, #8 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v9.h[6], w10 +; CHECK-GI-NEXT: lsl w10, w11, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #808] +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: sbfx w8, w8, #8, #8 +; CHECK-GI-NEXT: mov v11.h[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #816] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 ; CHECK-GI-NEXT: mov v17.s[1], wzr -; CHECK-GI-NEXT: mov v3.s[1], wzr -; CHECK-GI-NEXT: sxtb w12, w12 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: mov v9.h[7], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #824] +; CHECK-GI-NEXT: sbfx w16, w11, #8, #8 +; CHECK-GI-NEXT: mov v10.h[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #832] +; CHECK-GI-NEXT: mov v11.h[3], w13 +; CHECK-GI-NEXT: sbfx w15, w14, #8, #8 +; CHECK-GI-NEXT: lsl w14, w17, #8 +; CHECK-GI-NEXT: fmov s12, w16 +; CHECK-GI-NEXT: ldr w16, [sp, #872] +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #840] +; CHECK-GI-NEXT: sbfx w13, w18, #8, #8 +; CHECK-GI-NEXT: sbfx w17, w12, #8, #8 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: ldr w12, [sp, #856] +; CHECK-GI-NEXT: mov v12.h[1], w15 +; CHECK-GI-NEXT: mov v11.h[4], w14 +; CHECK-GI-NEXT: ldr w15, [sp, #880] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v10.h[6], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #848] +; CHECK-GI-NEXT: lsl w14, w15, #8 +; CHECK-GI-NEXT: sbfx w15, w16, #8, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #888] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: mov v20.s[1], wzr +; CHECK-GI-NEXT: mov v12.h[2], w17 +; CHECK-GI-NEXT: lsl w17, w10, #8 +; CHECK-GI-NEXT: mov v11.h[5], w9 +; CHECK-GI-NEXT: fmov s13, w15 +; CHECK-GI-NEXT: ldr w9, [sp, #936] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: sbfx w15, w17, #8, #8 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: lsl w9, w9, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: ldr w10, [sp, #864] +; CHECK-GI-NEXT: mov v12.h[3], w15 +; CHECK-GI-NEXT: mov v11.h[6], w8 +; CHECK-GI-NEXT: sbfx w8, w11, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #1000] +; CHECK-GI-NEXT: mov v13.h[1], w14 +; CHECK-GI-NEXT: ldr w15, [sp, #944] +; CHECK-GI-NEXT: sbfx w9, w9, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #896] +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: mov v12.h[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1008] +; CHECK-GI-NEXT: fmov s14, w9 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v13.h[2], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #952] +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: lsl w17, w8, #8 +; CHECK-GI-NEXT: smov w8, v29.h[3] +; CHECK-GI-NEXT: smov w9, v29.h[7] +; CHECK-GI-NEXT: fmov s29, w11 +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v14.h[1], w15 +; CHECK-GI-NEXT: sbfx w15, w17, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #904] +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: mov v12.h[5], w13 +; CHECK-GI-NEXT: mov v13.h[3], w14 +; CHECK-GI-NEXT: mov v29.h[1], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #960] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #1016] +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #1024] +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v14.h[2], w16 +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: ldr w16, [sp, #912] +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v13.h[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #968] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v12.h[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #976] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mov v14.h[3], w15 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v29.h[2], w14 +; CHECK-GI-NEXT: ldr w15, [sp, #1032] +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: ldr w14, [sp, #920] +; CHECK-GI-NEXT: mov v26.s[3], w8 +; CHECK-GI-NEXT: sbfx w16, w16, #8, #8 +; CHECK-GI-NEXT: lsl w15, w15, #8 +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: mov v14.h[4], w11 +; CHECK-GI-NEXT: mov v29.h[3], w13 +; CHECK-GI-NEXT: ldr w11, [sp, #984] +; CHECK-GI-NEXT: lsl w14, w14, #8 +; CHECK-GI-NEXT: sbfx w15, w15, #8, #8 +; CHECK-GI-NEXT: mov v13.h[5], w16 +; CHECK-GI-NEXT: ldr w16, [sp, #1040] +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #928] +; CHECK-GI-NEXT: sbfx w14, w14, #8, #8 +; CHECK-GI-NEXT: mov v12.h[7], w10 +; CHECK-GI-NEXT: mov v27.s[3], w9 +; CHECK-GI-NEXT: mov v14.h[5], w12 +; CHECK-GI-NEXT: mov v29.h[4], w15 +; CHECK-GI-NEXT: lsl w16, w16, #8 +; CHECK-GI-NEXT: sbfx w10, w11, #8, #8 +; CHECK-GI-NEXT: lsl w13, w13, #8 +; CHECK-GI-NEXT: mov v13.h[6], w14 +; CHECK-GI-NEXT: ldr w12, [sp, #1048] +; CHECK-GI-NEXT: sbfx w14, w16, #8, #8 +; CHECK-GI-NEXT: ldr w11, [sp, #728] +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: mul v15.8h, v8.8h, v12.8h +; CHECK-GI-NEXT: smov w16, v31.h[6] +; CHECK-GI-NEXT: mov v14.h[6], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #992] +; CHECK-GI-NEXT: mov v29.h[5], w14 +; CHECK-GI-NEXT: lsl w12, w12, #8 +; CHECK-GI-NEXT: lsl w11, w11, #8 +; CHECK-GI-NEXT: mov v13.h[7], w13 +; CHECK-GI-NEXT: lsl w10, w10, #8 +; CHECK-GI-NEXT: ldr w13, [sp, #792] +; CHECK-GI-NEXT: ldr w14, [sp, #1056] +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w11, w11, #8, #8 +; CHECK-GI-NEXT: mov v30.s[2], w16 +; CHECK-GI-NEXT: sbfx w10, w10, #8, #8 +; CHECK-GI-NEXT: smov w8, v15.h[1] +; CHECK-GI-NEXT: smov w9, v15.h[5] +; CHECK-GI-NEXT: mov v29.h[6], w12 +; CHECK-GI-NEXT: lsl w12, w13, #8 +; CHECK-GI-NEXT: lsl w13, w14, #8 +; CHECK-GI-NEXT: mov v10.h[7], w11 +; CHECK-GI-NEXT: mov v14.h[7], w10 +; CHECK-GI-NEXT: mul v12.8h, v9.8h, v13.8h +; CHECK-GI-NEXT: sbfx w12, w12, #8, #8 +; CHECK-GI-NEXT: sbfx w13, w13, #8, #8 +; CHECK-GI-NEXT: smov w10, v15.h[0] +; CHECK-GI-NEXT: smov w11, v15.h[4] +; CHECK-GI-NEXT: smov w14, v31.h[7] +; CHECK-GI-NEXT: smov w15, v31.h[3] +; CHECK-GI-NEXT: mov v11.h[7], w12 +; CHECK-GI-NEXT: mov v29.h[7], w13 ; CHECK-GI-NEXT: mov v6.s[1], wzr +; CHECK-GI-NEXT: mul v13.8h, v10.8h, v14.8h +; CHECK-GI-NEXT: smov w12, v12.h[0] +; CHECK-GI-NEXT: smov w13, v12.h[1] +; CHECK-GI-NEXT: mov v7.s[1], wzr ; CHECK-GI-NEXT: mov v2.s[1], wzr -; CHECK-GI-NEXT: mov v5.s[1], wzr ; CHECK-GI-NEXT: mov v4.s[1], wzr -; CHECK-GI-NEXT: mov v7.s[1], wzr -; CHECK-GI-NEXT: mov v10.s[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1080] -; CHECK-GI-NEXT: mov v8.s[1], wzr -; CHECK-GI-NEXT: mov v9.s[3], w9 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: ldr w10, [sp, #1032] -; CHECK-GI-NEXT: sxtb w9, w12 -; CHECK-GI-NEXT: mov v29.s[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1072] -; CHECK-GI-NEXT: mov v19.s[3], wzr -; CHECK-GI-NEXT: mov v21.s[3], wzr +; CHECK-GI-NEXT: fmov s31, w11 +; CHECK-GI-NEXT: mov v30.s[3], w14 +; CHECK-GI-NEXT: smov w11, v12.h[4] +; CHECK-GI-NEXT: mul v14.8h, v11.8h, v29.8h +; CHECK-GI-NEXT: fmov s29, w10 +; CHECK-GI-NEXT: smov w10, v15.h[2] +; CHECK-GI-NEXT: smov w14, v13.h[0] +; CHECK-GI-NEXT: fmov s8, w12 +; CHECK-GI-NEXT: smov w16, v13.h[1] +; CHECK-GI-NEXT: mov v31.s[1], w9 +; CHECK-GI-NEXT: smov w9, v12.h[2] +; CHECK-GI-NEXT: mov v28.s[3], w15 +; CHECK-GI-NEXT: mov v29.s[1], w8 +; CHECK-GI-NEXT: smov w8, v15.h[6] +; CHECK-GI-NEXT: smov w15, v12.h[5] +; CHECK-GI-NEXT: mov v8.s[1], w13 +; CHECK-GI-NEXT: fmov s9, w11 +; CHECK-GI-NEXT: smov w12, v15.h[3] +; CHECK-GI-NEXT: fmov s10, w14 +; CHECK-GI-NEXT: smov w14, v13.h[2] +; CHECK-GI-NEXT: smov w11, v12.h[6] +; CHECK-GI-NEXT: smov w13, v15.h[7] +; CHECK-GI-NEXT: mov v3.s[1], wzr +; CHECK-GI-NEXT: mov v5.s[1], wzr +; CHECK-GI-NEXT: mov v31.s[2], w8 +; CHECK-GI-NEXT: smov w8, v13.h[4] +; CHECK-GI-NEXT: mov v29.s[2], w10 +; CHECK-GI-NEXT: mov v10.s[1], w16 +; CHECK-GI-NEXT: smov w16, v14.h[0] +; CHECK-GI-NEXT: mov v8.s[2], w9 +; CHECK-GI-NEXT: smov w9, v13.h[5] +; CHECK-GI-NEXT: smov w10, v12.h[3] +; CHECK-GI-NEXT: mov v9.s[1], w15 +; CHECK-GI-NEXT: smov w15, v13.h[6] ; CHECK-GI-NEXT: mov v1.s[1], wzr -; CHECK-GI-NEXT: mul w8, w8, w9 +; CHECK-GI-NEXT: mov v0.s[1], wzr +; CHECK-GI-NEXT: fmov s11, w8 +; CHECK-GI-NEXT: smov w8, v14.h[1] +; CHECK-GI-NEXT: mov v29.s[3], w12 +; CHECK-GI-NEXT: mov v10.s[2], w14 +; CHECK-GI-NEXT: smov w14, v12.h[7] +; CHECK-GI-NEXT: fmov s12, w16 +; CHECK-GI-NEXT: smov w12, v14.h[4] +; CHECK-GI-NEXT: mov v8.s[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #536] +; CHECK-GI-NEXT: mov v11.s[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #272] +; CHECK-GI-NEXT: mov v9.s[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #800] +; CHECK-GI-NEXT: mov v12.s[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1064] +; CHECK-GI-NEXT: mov v31.s[3], w13 +; CHECK-GI-NEXT: smov w13, v14.h[5] +; CHECK-GI-NEXT: sxtb w9, w9 ; CHECK-GI-NEXT: sxtb w10, w10 ; CHECK-GI-NEXT: sxtb w11, w11 +; CHECK-GI-NEXT: sxtb w8, w8 +; CHECK-GI-NEXT: mov v11.s[2], w15 +; CHECK-GI-NEXT: smov w15, v13.h[3] +; CHECK-GI-NEXT: smov w16, v13.h[7] +; CHECK-GI-NEXT: fmov s13, w12 +; CHECK-GI-NEXT: mul w9, w9, w10 +; CHECK-GI-NEXT: smov w12, v14.h[2] +; CHECK-GI-NEXT: mul w8, w11, w8 +; CHECK-GI-NEXT: mov v19.s[2], wzr +; CHECK-GI-NEXT: mov v21.s[2], wzr ; CHECK-GI-NEXT: mov v16.s[2], wzr +; CHECK-GI-NEXT: mov v18.s[2], wzr ; CHECK-GI-NEXT: mov v17.s[2], wzr -; CHECK-GI-NEXT: mov v3.s[2], wzr +; CHECK-GI-NEXT: mov v13.s[1], w13 +; CHECK-GI-NEXT: smov w13, v14.h[6] +; CHECK-GI-NEXT: sxth w9, w9 +; CHECK-GI-NEXT: sxth w10, w8 +; CHECK-GI-NEXT: mov v20.s[2], wzr ; CHECK-GI-NEXT: mov v6.s[2], wzr +; CHECK-GI-NEXT: mov v7.s[2], wzr ; CHECK-GI-NEXT: mov v2.s[2], wzr -; CHECK-GI-NEXT: mov v5.s[2], wzr ; CHECK-GI-NEXT: mov v4.s[2], wzr -; CHECK-GI-NEXT: mov v7.s[2], wzr -; CHECK-GI-NEXT: mov v8.s[2], wzr -; CHECK-GI-NEXT: mov v29.s[2], w10 -; CHECK-GI-NEXT: mov v10.s[3], w11 -; CHECK-GI-NEXT: add v19.4s, v19.4s, v21.4s -; CHECK-GI-NEXT: ldr w9, [sp, #976] -; CHECK-GI-NEXT: fmov s21, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1040] +; CHECK-GI-NEXT: mov v3.s[2], wzr +; CHECK-GI-NEXT: mov v5.s[2], wzr +; CHECK-GI-NEXT: add v22.4s, v22.4s, v23.4s +; CHECK-GI-NEXT: add v25.4s, v24.4s, v25.4s +; CHECK-GI-NEXT: fmov s23, w9 +; CHECK-GI-NEXT: fmov s24, w10 +; CHECK-GI-NEXT: mov v12.s[2], w12 +; CHECK-GI-NEXT: mov v13.s[2], w13 +; CHECK-GI-NEXT: smov w8, v14.h[3] +; CHECK-GI-NEXT: smov w9, v14.h[7] ; CHECK-GI-NEXT: mov v1.s[2], wzr +; CHECK-GI-NEXT: mov v0.s[2], wzr +; CHECK-GI-NEXT: mov v19.s[3], wzr +; CHECK-GI-NEXT: mov v21.s[3], wzr ; CHECK-GI-NEXT: mov v16.s[3], wzr +; CHECK-GI-NEXT: mov v18.s[3], wzr ; CHECK-GI-NEXT: mov v17.s[3], wzr -; CHECK-GI-NEXT: sxtb w9, w9 -; CHECK-GI-NEXT: sxtb w8, w8 -; CHECK-GI-NEXT: mov v11.16b, v8.16b -; CHECK-GI-NEXT: mov v3.s[3], wzr +; CHECK-GI-NEXT: mov v20.s[3], wzr ; CHECK-GI-NEXT: mov v6.s[3], wzr +; CHECK-GI-NEXT: mov v7.s[3], wzr ; CHECK-GI-NEXT: mov v2.s[3], wzr -; CHECK-GI-NEXT: mov v5.s[3], wzr ; CHECK-GI-NEXT: mov v4.s[3], wzr -; CHECK-GI-NEXT: mov v7.s[3], wzr -; CHECK-GI-NEXT: mov v25.s[1], wzr -; CHECK-GI-NEXT: mov v21.s[1], wzr -; CHECK-GI-NEXT: mul v8.4s, v13.4s, v9.4s -; CHECK-GI-NEXT: mul v9.4s, v14.4s, v10.4s -; CHECK-GI-NEXT: mov v23.s[3], w9 -; CHECK-GI-NEXT: mov v29.s[3], w8 +; CHECK-GI-NEXT: mov v3.s[3], wzr +; CHECK-GI-NEXT: mov v5.s[3], wzr +; CHECK-GI-NEXT: mov v23.s[1], wzr +; CHECK-GI-NEXT: mov v24.s[1], wzr +; CHECK-GI-NEXT: mov v9.s[3], w14 +; CHECK-GI-NEXT: mov v10.s[3], w15 +; CHECK-GI-NEXT: mov v11.s[3], w16 ; CHECK-GI-NEXT: mov v1.s[3], wzr -; CHECK-GI-NEXT: mov v11.s[3], wzr -; CHECK-GI-NEXT: add v16.4s, v16.4s, v17.4s -; CHECK-GI-NEXT: add v3.4s, v3.4s, v6.4s -; CHECK-GI-NEXT: add v2.4s, v2.4s, v5.4s -; CHECK-GI-NEXT: add v4.4s, v4.4s, v7.4s -; CHECK-GI-NEXT: mov v25.s[2], wzr -; CHECK-GI-NEXT: mov v21.s[2], wzr -; CHECK-GI-NEXT: mla v20.4s, v28.4s, v22.4s -; CHECK-GI-NEXT: mla v8.4s, v31.4s, v23.4s -; CHECK-GI-NEXT: mla v9.4s, v12.4s, v29.4s -; CHECK-GI-NEXT: add v5.4s, v19.4s, v16.4s -; CHECK-GI-NEXT: add v1.4s, v1.4s, v18.4s -; CHECK-GI-NEXT: add v3.4s, v11.4s, v3.4s +; CHECK-GI-NEXT: mov v12.s[3], w8 +; CHECK-GI-NEXT: mov v13.s[3], w9 +; CHECK-GI-NEXT: mov v0.s[3], wzr +; CHECK-GI-NEXT: add v19.4s, v19.4s, v21.4s +; CHECK-GI-NEXT: add v16.4s, v16.4s, v18.4s +; CHECK-GI-NEXT: add v17.4s, v17.4s, v20.4s +; CHECK-GI-NEXT: add v6.4s, v6.4s, v7.4s ; CHECK-GI-NEXT: add v2.4s, v2.4s, v4.4s -; CHECK-GI-NEXT: add v4.4s, v27.4s, v30.4s -; CHECK-GI-NEXT: add v6.4s, v24.4s, v26.4s -; CHECK-GI-NEXT: ldr x29, [sp, #80] // 8-byte Folded Reload -; CHECK-GI-NEXT: mov v25.s[3], wzr -; CHECK-GI-NEXT: mov v21.s[3], wzr -; CHECK-GI-NEXT: add v0.4s, v0.4s, v20.4s -; CHECK-GI-NEXT: add v1.4s, v1.4s, v5.4s -; CHECK-GI-NEXT: add v5.4s, v8.4s, v9.4s -; CHECK-GI-NEXT: add v2.4s, v3.4s, v2.4s -; CHECK-GI-NEXT: add v3.4s, v4.4s, v6.4s -; CHECK-GI-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload -; CHECK-GI-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload -; CHECK-GI-NEXT: add v1.4s, v25.4s, v1.4s -; CHECK-GI-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-GI-NEXT: add v2.4s, v21.4s, v2.4s -; CHECK-GI-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload -; CHECK-GI-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-GI-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-GI-NEXT: add v3.4s, v3.4s, v5.4s +; CHECK-GI-NEXT: mov v23.s[2], wzr +; CHECK-GI-NEXT: mov v24.s[2], wzr +; CHECK-GI-NEXT: add v26.4s, v26.4s, v27.4s +; CHECK-GI-NEXT: add v27.4s, v28.4s, v30.4s +; CHECK-GI-NEXT: add v1.4s, v1.4s, v19.4s +; CHECK-GI-NEXT: add v4.4s, v16.4s, v17.4s +; CHECK-GI-NEXT: add v5.4s, v29.4s, v31.4s +; CHECK-GI-NEXT: add v7.4s, v8.4s, v9.4s +; CHECK-GI-NEXT: add v16.4s, v10.4s, v11.4s +; CHECK-GI-NEXT: add v17.4s, v12.4s, v13.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v6.4s +; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: mov v23.s[3], wzr +; CHECK-GI-NEXT: mov v24.s[3], wzr +; CHECK-GI-NEXT: add v3.4s, v22.4s, v25.4s +; CHECK-GI-NEXT: add v6.4s, v26.4s, v27.4s +; CHECK-GI-NEXT: add v1.4s, v1.4s, v4.4s +; CHECK-GI-NEXT: add v4.4s, v5.4s, v7.4s +; CHECK-GI-NEXT: add v5.4s, v16.4s, v17.4s ; CHECK-GI-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: ldr x29, [sp, #64] // 8-byte Folded Reload +; CHECK-GI-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: add v2.4s, v3.4s, v6.4s +; CHECK-GI-NEXT: add v1.4s, v23.4s, v1.4s +; CHECK-GI-NEXT: add v3.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: add v0.4s, v24.4s, v0.4s +; CHECK-GI-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-GI-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s ; CHECK-GI-NEXT: addv s1, v1.4s ; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: fmov w8, s1 ; CHECK-GI-NEXT: fmov w9, s0 ; CHECK-GI-NEXT: add w0, w8, w9 -; CHECK-GI-NEXT: add sp, sp, #96 +; CHECK-GI-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-GI-NEXT: ret entry: %az = sext <33 x i8> %a to <33 x i32> diff --git a/llvm/test/CodeGen/AArch64/neon-extmul.ll b/llvm/test/CodeGen/AArch64/neon-extmul.ll index c82f8e1..84b634d 100644 --- a/llvm/test/CodeGen/AArch64/neon-extmul.ll +++ b/llvm/test/CodeGen/AArch64/neon-extmul.ll @@ -12,10 +12,9 @@ define <8 x i32> @extmuls_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) { ; ; CHECK-GI-LABEL: extmuls_v8i8_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: smull v0.4s, v2.4h, v1.4h -; CHECK-GI-NEXT: smull2 v1.4s, v2.8h, v1.8h +; CHECK-GI-NEXT: smull v1.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: sshll v0.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 ; CHECK-GI-NEXT: ret entry: %s0s = sext <8 x i8> %s0 to <8 x i32> @@ -34,10 +33,9 @@ define <8 x i32> @extmulu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) { ; ; CHECK-GI-LABEL: extmulu_v8i8_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: umull v0.4s, v2.4h, v1.4h -; CHECK-GI-NEXT: umull2 v1.4s, v2.8h, v1.8h +; CHECK-GI-NEXT: umull v1.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 ; CHECK-GI-NEXT: ret entry: %s0s = zext <8 x i8> %s0 to <8 x i32> @@ -79,12 +77,9 @@ define <8 x i32> @extmuladds_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) ; ; CHECK-GI-LABEL: extmuladds_v8i8_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: smlal v2.4s, v0.4h, v1.4h -; CHECK-GI-NEXT: smlal2 v3.4s, v0.8h, v1.8h -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: mov v1.16b, v3.16b +; CHECK-GI-NEXT: smull v1.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: saddw v0.4s, v2.4s, v1.4h +; CHECK-GI-NEXT: saddw2 v1.4s, v3.4s, v1.8h ; CHECK-GI-NEXT: ret entry: %s0s = sext <8 x i8> %s0 to <8 x i32> @@ -104,12 +99,9 @@ define <8 x i32> @extmuladdu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) ; ; CHECK-GI-LABEL: extmuladdu_v8i8_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: umlal v2.4s, v0.4h, v1.4h -; CHECK-GI-NEXT: umlal2 v3.4s, v0.8h, v1.8h -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: mov v1.16b, v3.16b +; CHECK-GI-NEXT: umull v1.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: uaddw v0.4s, v2.4s, v1.4h +; CHECK-GI-NEXT: uaddw2 v1.4s, v3.4s, v1.8h ; CHECK-GI-NEXT: ret entry: %s0s = zext <8 x i8> %s0 to <8 x i32> @@ -163,16 +155,13 @@ define <8 x i64> @extmuls_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) { ; ; CHECK-GI-LABEL: extmuls_v8i8_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-NEXT: sshll2 v4.4s, v0.8h, #0 -; CHECK-GI-NEXT: sshll2 v5.4s, v1.8h, #0 -; CHECK-GI-NEXT: smull v0.2d, v2.2s, v3.2s -; CHECK-GI-NEXT: smull2 v1.2d, v2.4s, v3.4s -; CHECK-GI-NEXT: smull v2.2d, v4.2s, v5.2s -; CHECK-GI-NEXT: smull2 v3.2d, v4.4s, v5.4s +; CHECK-GI-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v3.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v0.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll2 v1.2d, v1.4s, #0 +; CHECK-GI-NEXT: sshll v2.2d, v3.2s, #0 +; CHECK-GI-NEXT: sshll2 v3.2d, v3.4s, #0 ; CHECK-GI-NEXT: ret entry: %s0s = sext <8 x i8> %s0 to <8 x i64> @@ -195,16 +184,13 @@ define <8 x i64> @extmulu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) { ; ; CHECK-GI-LABEL: extmulu_v8i8_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-NEXT: ushll2 v4.4s, v0.8h, #0 -; CHECK-GI-NEXT: ushll2 v5.4s, v1.8h, #0 -; CHECK-GI-NEXT: umull v0.2d, v2.2s, v3.2s -; CHECK-GI-NEXT: umull2 v1.2d, v2.4s, v3.4s -; CHECK-GI-NEXT: umull v2.2d, v4.2s, v5.2s -; CHECK-GI-NEXT: umull2 v3.2d, v4.4s, v5.4s +; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0 +; CHECK-GI-NEXT: ushll v2.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll2 v3.2d, v3.4s, #0 ; CHECK-GI-NEXT: ret entry: %s0s = zext <8 x i8> %s0 to <8 x i64> @@ -263,20 +249,13 @@ define <8 x i64> @extmuladds_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) ; ; CHECK-GI-LABEL: extmuladds_v8i8_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: sshll v6.4s, v0.4h, #0 -; CHECK-GI-NEXT: sshll v7.4s, v1.4h, #0 -; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: smlal v2.2d, v6.2s, v7.2s -; CHECK-GI-NEXT: smlal2 v3.2d, v6.4s, v7.4s -; CHECK-GI-NEXT: smlal v4.2d, v0.2s, v1.2s -; CHECK-GI-NEXT: smlal2 v5.2d, v0.4s, v1.4s -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: mov v1.16b, v3.16b -; CHECK-GI-NEXT: mov v2.16b, v4.16b -; CHECK-GI-NEXT: mov v3.16b, v5.16b +; CHECK-GI-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v0.8h, #0 +; CHECK-GI-NEXT: saddw v0.2d, v2.2d, v1.2s +; CHECK-GI-NEXT: saddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: saddw v2.2d, v4.2d, v6.2s +; CHECK-GI-NEXT: saddw2 v3.2d, v5.2d, v6.4s ; CHECK-GI-NEXT: ret entry: %s0s = sext <8 x i8> %s0 to <8 x i64> @@ -301,20 +280,13 @@ define <8 x i64> @extmuladdu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) ; ; CHECK-GI-LABEL: extmuladdu_v8i8_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll v6.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll v7.4s, v1.4h, #0 -; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: umlal v2.2d, v6.2s, v7.2s -; CHECK-GI-NEXT: umlal2 v3.2d, v6.4s, v7.4s -; CHECK-GI-NEXT: umlal v4.2d, v0.2s, v1.2s -; CHECK-GI-NEXT: umlal2 v5.2d, v0.4s, v1.4s -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: mov v1.16b, v3.16b -; CHECK-GI-NEXT: mov v2.16b, v4.16b -; CHECK-GI-NEXT: mov v3.16b, v5.16b +; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v6.4s, v0.8h, #0 +; CHECK-GI-NEXT: uaddw v0.2d, v2.2d, v1.2s +; CHECK-GI-NEXT: uaddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: uaddw v2.2d, v4.2d, v6.2s +; CHECK-GI-NEXT: uaddw2 v3.2d, v5.2d, v6.4s ; CHECK-GI-NEXT: ret entry: %s0s = zext <8 x i8> %s0 to <8 x i64> diff --git a/llvm/test/CodeGen/AArch64/peephole-and-tst.ll b/llvm/test/CodeGen/AArch64/peephole-and-tst.ll index 17ad298..3caac1d 100644 --- a/llvm/test/CodeGen/AArch64/peephole-and-tst.ll +++ b/llvm/test/CodeGen/AArch64/peephole-and-tst.ll @@ -1,40 +1,72 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s +; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64-- -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI %struct.anon = type { ptr, ptr } @ptr_wrapper = common global ptr null, align 8 define i32 @test_func_i32_two_uses(i32 %in, i32 %bit, i32 %mask) { -; CHECK-LABEL: test_func_i32_two_uses: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: adrp x8, :got:ptr_wrapper -; CHECK-NEXT: ldr x8, [x8, :got_lo12:ptr_wrapper] -; CHECK-NEXT: ldr x9, [x8] -; CHECK-NEXT: mov w8, wzr -; CHECK-NEXT: b .LBB0_3 -; CHECK-NEXT: .LBB0_1: // in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: str xzr, [x9, #8] -; CHECK-NEXT: .LBB0_2: // in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: lsl w1, w1, #1 -; CHECK-NEXT: cbz w1, .LBB0_6 -; CHECK-NEXT: .LBB0_3: // %do.body -; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ands w10, w1, w0 -; CHECK-NEXT: and w11, w2, w0 -; CHECK-NEXT: cinc w8, w8, ne -; CHECK-NEXT: cmp w10, w11 -; CHECK-NEXT: b.eq .LBB0_1 -; CHECK-NEXT: // %bb.4: // %do.body -; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: cbnz w2, .LBB0_1 -; CHECK-NEXT: // %bb.5: // %do.body -; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1 -; CHECK-NEXT: cbz w10, .LBB0_2 -; CHECK-NEXT: b .LBB0_1 -; CHECK-NEXT: .LBB0_6: // %do.end -; CHECK-NEXT: mov w0, w8 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_func_i32_two_uses: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: adrp x8, :got:ptr_wrapper +; CHECK-SD-NEXT: ldr x8, [x8, :got_lo12:ptr_wrapper] +; CHECK-SD-NEXT: ldr x9, [x8] +; CHECK-SD-NEXT: mov w8, wzr +; CHECK-SD-NEXT: b .LBB0_3 +; CHECK-SD-NEXT: .LBB0_1: // in Loop: Header=BB0_3 Depth=1 +; CHECK-SD-NEXT: str xzr, [x9, #8] +; CHECK-SD-NEXT: .LBB0_2: // in Loop: Header=BB0_3 Depth=1 +; CHECK-SD-NEXT: lsl w1, w1, #1 +; CHECK-SD-NEXT: cbz w1, .LBB0_6 +; CHECK-SD-NEXT: .LBB0_3: // %do.body +; CHECK-SD-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-SD-NEXT: ands w10, w1, w0 +; CHECK-SD-NEXT: and w11, w2, w0 +; CHECK-SD-NEXT: cinc w8, w8, ne +; CHECK-SD-NEXT: cmp w10, w11 +; CHECK-SD-NEXT: b.eq .LBB0_1 +; CHECK-SD-NEXT: // %bb.4: // %do.body +; CHECK-SD-NEXT: // in Loop: Header=BB0_3 Depth=1 +; CHECK-SD-NEXT: cbnz w2, .LBB0_1 +; CHECK-SD-NEXT: // %bb.5: // %do.body +; CHECK-SD-NEXT: // in Loop: Header=BB0_3 Depth=1 +; CHECK-SD-NEXT: cbz w10, .LBB0_2 +; CHECK-SD-NEXT: b .LBB0_1 +; CHECK-SD-NEXT: .LBB0_6: // %do.end +; CHECK-SD-NEXT: mov w0, w8 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_func_i32_two_uses: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: adrp x8, :got:ptr_wrapper +; CHECK-GI-NEXT: ldr x8, [x8, :got_lo12:ptr_wrapper] +; CHECK-GI-NEXT: ldr x9, [x8] +; CHECK-GI-NEXT: mov w8, wzr +; CHECK-GI-NEXT: b .LBB0_3 +; CHECK-GI-NEXT: .LBB0_1: // in Loop: Header=BB0_3 Depth=1 +; CHECK-GI-NEXT: str xzr, [x9, #8] +; CHECK-GI-NEXT: .LBB0_2: // in Loop: Header=BB0_3 Depth=1 +; CHECK-GI-NEXT: lsl w1, w1, #1 +; CHECK-GI-NEXT: cbz w1, .LBB0_6 +; CHECK-GI-NEXT: .LBB0_3: // %do.body +; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-GI-NEXT: and w10, w1, w0 +; CHECK-GI-NEXT: tst w1, w0 +; CHECK-GI-NEXT: and w11, w2, w0 +; CHECK-GI-NEXT: cinc w8, w8, ne +; CHECK-GI-NEXT: cmp w10, w11 +; CHECK-GI-NEXT: b.eq .LBB0_1 +; CHECK-GI-NEXT: // %bb.4: // %do.body +; CHECK-GI-NEXT: // in Loop: Header=BB0_3 Depth=1 +; CHECK-GI-NEXT: cbnz w2, .LBB0_1 +; CHECK-GI-NEXT: // %bb.5: // %do.body +; CHECK-GI-NEXT: // in Loop: Header=BB0_3 Depth=1 +; CHECK-GI-NEXT: cbz w10, .LBB0_2 +; CHECK-GI-NEXT: b .LBB0_1 +; CHECK-GI-NEXT: .LBB0_6: // %do.end +; CHECK-GI-NEXT: mov w0, w8 +; CHECK-GI-NEXT: ret entry: %0 = load ptr, ptr @ptr_wrapper, align 8 %result = getelementptr inbounds %struct.anon, ptr %0, i64 0, i32 1 @@ -70,28 +102,52 @@ do.end: ; preds = %4 } define i32 @test_func_i64_one_use(i64 %in, i64 %bit, i64 %mask) { -; CHECK-LABEL: test_func_i64_one_use: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: adrp x8, :got:ptr_wrapper -; CHECK-NEXT: ldr x8, [x8, :got_lo12:ptr_wrapper] -; CHECK-NEXT: ldr x9, [x8] -; CHECK-NEXT: mov w8, wzr -; CHECK-NEXT: b .LBB1_2 -; CHECK-NEXT: .LBB1_1: // in Loop: Header=BB1_2 Depth=1 -; CHECK-NEXT: lsl x1, x1, #1 -; CHECK-NEXT: cbz x1, .LBB1_4 -; CHECK-NEXT: .LBB1_2: // %do.body -; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ands x10, x1, x0 -; CHECK-NEXT: orr x10, x2, x10 -; CHECK-NEXT: cinc w8, w8, ne -; CHECK-NEXT: cbz x10, .LBB1_1 -; CHECK-NEXT: // %bb.3: // in Loop: Header=BB1_2 Depth=1 -; CHECK-NEXT: str xzr, [x9, #8] -; CHECK-NEXT: b .LBB1_1 -; CHECK-NEXT: .LBB1_4: // %do.end -; CHECK-NEXT: mov w0, w8 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_func_i64_one_use: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: adrp x8, :got:ptr_wrapper +; CHECK-SD-NEXT: ldr x8, [x8, :got_lo12:ptr_wrapper] +; CHECK-SD-NEXT: ldr x9, [x8] +; CHECK-SD-NEXT: mov w8, wzr +; CHECK-SD-NEXT: b .LBB1_2 +; CHECK-SD-NEXT: .LBB1_1: // in Loop: Header=BB1_2 Depth=1 +; CHECK-SD-NEXT: lsl x1, x1, #1 +; CHECK-SD-NEXT: cbz x1, .LBB1_4 +; CHECK-SD-NEXT: .LBB1_2: // %do.body +; CHECK-SD-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-SD-NEXT: ands x10, x1, x0 +; CHECK-SD-NEXT: orr x10, x2, x10 +; CHECK-SD-NEXT: cinc w8, w8, ne +; CHECK-SD-NEXT: cbz x10, .LBB1_1 +; CHECK-SD-NEXT: // %bb.3: // in Loop: Header=BB1_2 Depth=1 +; CHECK-SD-NEXT: str xzr, [x9, #8] +; CHECK-SD-NEXT: b .LBB1_1 +; CHECK-SD-NEXT: .LBB1_4: // %do.end +; CHECK-SD-NEXT: mov w0, w8 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_func_i64_one_use: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: adrp x8, :got:ptr_wrapper +; CHECK-GI-NEXT: ldr x8, [x8, :got_lo12:ptr_wrapper] +; CHECK-GI-NEXT: ldr x9, [x8] +; CHECK-GI-NEXT: mov w8, wzr +; CHECK-GI-NEXT: b .LBB1_2 +; CHECK-GI-NEXT: .LBB1_1: // in Loop: Header=BB1_2 Depth=1 +; CHECK-GI-NEXT: lsl x1, x1, #1 +; CHECK-GI-NEXT: cbz x1, .LBB1_4 +; CHECK-GI-NEXT: .LBB1_2: // %do.body +; CHECK-GI-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-GI-NEXT: and x10, x1, x0 +; CHECK-GI-NEXT: tst x1, x0 +; CHECK-GI-NEXT: orr x10, x2, x10 +; CHECK-GI-NEXT: cinc w8, w8, ne +; CHECK-GI-NEXT: cbz x10, .LBB1_1 +; CHECK-GI-NEXT: // %bb.3: // in Loop: Header=BB1_2 Depth=1 +; CHECK-GI-NEXT: str xzr, [x9, #8] +; CHECK-GI-NEXT: b .LBB1_1 +; CHECK-GI-NEXT: .LBB1_4: // %do.end +; CHECK-GI-NEXT: mov w0, w8 +; CHECK-GI-NEXT: ret entry: %0 = load ptr, ptr @ptr_wrapper, align 8 %result = getelementptr inbounds %struct.anon, ptr %0, i64 0, i32 1 @@ -124,11 +180,18 @@ do.end: ; preds = %4 } define i64 @test_and1(i64 %x, i64 %y) { -; CHECK-LABEL: test_and1: -; CHECK: // %bb.0: -; CHECK-NEXT: ands x8, x0, #0x3 -; CHECK-NEXT: csel x0, x8, x1, eq -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_and1: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: ands x8, x0, #0x3 +; CHECK-SD-NEXT: csel x0, x8, x1, eq +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_and1: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: and x8, x0, #0x3 +; CHECK-GI-NEXT: tst x0, #0x3 +; CHECK-GI-NEXT: csel x0, x8, x1, eq +; CHECK-GI-NEXT: ret %a = and i64 %x, 3 %c = icmp eq i64 %a, 0 %s = select i1 %c, i64 %a, i64 %y @@ -148,23 +211,43 @@ define i64 @test_and2(i64 %x, i64 %y) { } define i64 @test_and3(i64 %x, i64 %y) { -; CHECK-LABEL: test_and3: -; CHECK: // %bb.0: -; CHECK-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill -; CHECK-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 32 -; CHECK-NEXT: .cfi_offset w19, -8 -; CHECK-NEXT: .cfi_offset w20, -16 -; CHECK-NEXT: .cfi_offset w30, -32 -; CHECK-NEXT: mov x20, x0 -; CHECK-NEXT: mov x0, xzr -; CHECK-NEXT: mov x19, x1 -; CHECK-NEXT: bl callee -; CHECK-NEXT: ands x8, x20, #0x3 -; CHECK-NEXT: csel x0, x8, x19, eq -; CHECK-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_and3: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill +; CHECK-SD-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; CHECK-SD-NEXT: .cfi_def_cfa_offset 32 +; CHECK-SD-NEXT: .cfi_offset w19, -8 +; CHECK-SD-NEXT: .cfi_offset w20, -16 +; CHECK-SD-NEXT: .cfi_offset w30, -32 +; CHECK-SD-NEXT: mov x20, x0 +; CHECK-SD-NEXT: mov x0, xzr +; CHECK-SD-NEXT: mov x19, x1 +; CHECK-SD-NEXT: bl callee +; CHECK-SD-NEXT: ands x8, x20, #0x3 +; CHECK-SD-NEXT: csel x0, x8, x19, eq +; CHECK-SD-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; CHECK-SD-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_and3: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: stp x30, x21, [sp, #-32]! // 16-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 32 +; CHECK-GI-NEXT: .cfi_offset w19, -8 +; CHECK-GI-NEXT: .cfi_offset w20, -16 +; CHECK-GI-NEXT: .cfi_offset w21, -24 +; CHECK-GI-NEXT: .cfi_offset w30, -32 +; CHECK-GI-NEXT: mov x19, x0 +; CHECK-GI-NEXT: and x21, x0, #0x3 +; CHECK-GI-NEXT: mov x0, xzr +; CHECK-GI-NEXT: mov x20, x1 +; CHECK-GI-NEXT: bl callee +; CHECK-GI-NEXT: tst x19, #0x3 +; CHECK-GI-NEXT: csel x0, x21, x20, eq +; CHECK-GI-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: ldp x30, x21, [sp], #32 // 16-byte Folded Reload +; CHECK-GI-NEXT: ret %a = and i64 %x, 3 %b = call i64 @callee(i64 0) %c = icmp eq i64 %a, 0 @@ -173,19 +256,37 @@ define i64 @test_and3(i64 %x, i64 %y) { } define i64 @test_and_4(i64 %x, i64 %y) { -; CHECK-LABEL: test_and_4: -; CHECK: // %bb.0: -; CHECK-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w19, -8 -; CHECK-NEXT: .cfi_offset w30, -16 -; CHECK-NEXT: mov x19, x0 -; CHECK-NEXT: ands x0, x0, #0x3 -; CHECK-NEXT: bl callee -; CHECK-NEXT: ands x8, x19, #0x3 -; CHECK-NEXT: csel x0, x8, x0, eq -; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload -; CHECK-NEXT: ret +; CHECK-SD-LABEL: test_and_4: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill +; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 +; CHECK-SD-NEXT: .cfi_offset w19, -8 +; CHECK-SD-NEXT: .cfi_offset w30, -16 +; CHECK-SD-NEXT: mov x19, x0 +; CHECK-SD-NEXT: ands x0, x0, #0x3 +; CHECK-SD-NEXT: bl callee +; CHECK-SD-NEXT: ands x8, x19, #0x3 +; CHECK-SD-NEXT: csel x0, x8, x0, eq +; CHECK-SD-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_and_4: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: str x30, [sp, #-32]! // 8-byte Folded Spill +; CHECK-GI-NEXT: stp x20, x19, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 32 +; CHECK-GI-NEXT: .cfi_offset w19, -8 +; CHECK-GI-NEXT: .cfi_offset w20, -16 +; CHECK-GI-NEXT: .cfi_offset w30, -32 +; CHECK-GI-NEXT: and x20, x0, #0x3 +; CHECK-GI-NEXT: mov x19, x0 +; CHECK-GI-NEXT: mov x0, x20 +; CHECK-GI-NEXT: bl callee +; CHECK-GI-NEXT: tst x19, #0x3 +; CHECK-GI-NEXT: csel x0, x20, x0, eq +; CHECK-GI-NEXT: ldp x20, x19, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: ldr x30, [sp], #32 // 8-byte Folded Reload +; CHECK-GI-NEXT: ret %a = and i64 %x, 3 %b = call i64 @callee(i64 %a) %c = icmp eq i64 %a, 0 diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll index 52cb2d9..c7fb2db 100644 --- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll @@ -267,7 +267,7 @@ define <vscale x 32 x i16> @interleave4_nxv8i16(<vscale x 8 x i16> %vec0, <vscal ; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3 ; SME2-NEXT: zip { z0.h - z3.h }, { z0.h - z3.h } ; SME2-NEXT: ret - %retval = call <vscale x 32 x i16> @llvm.vector.interleave4.nxv8i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3) + %retval = call <vscale x 32 x i16> @llvm.vector.interleave4.nxv32i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3) ret <vscale x 32 x i16> %retval } @@ -540,30 +540,81 @@ define <vscale x 4 x i32> @interleave2_nxv2i32(<vscale x 2 x i32> %vec0, <vscale ret <vscale x 4 x i32> %retval } -; Float declarations -declare <vscale x 4 x half> @llvm.vector.interleave2.nxv4f16(<vscale x 2 x half>, <vscale x 2 x half>) -declare <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half>, <vscale x 4 x half>) -declare <vscale x 16 x half> @llvm.vector.interleave2.nxv16f16(<vscale x 8 x half>, <vscale x 8 x half>) -declare <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float>, <vscale x 2 x float>) -declare <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float>, <vscale x 4 x float>) -declare <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double>, <vscale x 2 x double>) +define <vscale x 4 x i16> @interleave2_same_const_splat_nxv4i16() { +; CHECK-LABEL: interleave2_same_const_splat_nxv4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z0.s, #3 // =0x3 +; CHECK-NEXT: ret + %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3)) + ret <vscale x 4 x i16> %retval +} + +define <vscale x 4 x i16> @interleave2_diff_const_splat_nxv4i16() { +; SVE-LABEL: interleave2_diff_const_splat_nxv4i16: +; SVE: // %bb.0: +; SVE-NEXT: mov z0.d, #4 // =0x4 +; SVE-NEXT: mov z1.d, #3 // =0x3 +; SVE-NEXT: zip2 z2.d, z1.d, z0.d +; SVE-NEXT: zip1 z0.d, z1.d, z0.d +; SVE-NEXT: uzp1 z0.s, z0.s, z2.s +; SVE-NEXT: ret +; +; SME2-LABEL: interleave2_diff_const_splat_nxv4i16: +; SME2: // %bb.0: +; SME2-NEXT: mov z0.d, #4 // =0x4 +; SME2-NEXT: mov z1.d, #3 // =0x3 +; SME2-NEXT: zip { z0.d, z1.d }, z1.d, z0.d +; SME2-NEXT: uzp1 z0.s, z0.s, z1.s +; SME2-NEXT: ret + %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.v4i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 4)) + ret <vscale x 4 x i16> %retval +} -; Integer declarations -declare <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8>, <vscale x 16 x i8>) -declare <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16>, <vscale x 8 x i16>) -declare <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32>, <vscale x 4 x i32>) -declare <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64>, <vscale x 2 x i64>) +define <vscale x 4 x i16> @interleave2_same_nonconst_splat_nxv4i16(i16 %a) { +; CHECK-LABEL: interleave2_same_nonconst_splat_nxv4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ret + %ins = insertelement <vscale x 2 x i16> poison, i16 %a, i32 0 + %splat = shufflevector <vscale x 2 x i16> %ins, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer + %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %splat, <vscale x 2 x i16> %splat) + ret <vscale x 4 x i16> %retval +} -; Predicated -declare <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1>, <vscale x 16 x i1>) -declare <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1>, <vscale x 8 x i1>) -declare <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1>, <vscale x 4 x i1>) -declare <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1>, <vscale x 2 x i1>) - -; Illegal type size -declare <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32>, <vscale x 8 x i32>) -declare <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64>, <vscale x 4 x i64>) - -declare <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8>, <vscale x 8 x i8>) -declare <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16>, <vscale x 4 x i16>) -declare <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32>, <vscale x 2 x i32>) +define <vscale x 4 x i16> @interleave2_diff_nonconst_splat_nxv4i16(i16 %a, i16 %b) { +; SVE-LABEL: interleave2_diff_nonconst_splat_nxv4i16: +; SVE: // %bb.0: +; SVE-NEXT: // kill: def $w1 killed $w1 def $x1 +; SVE-NEXT: // kill: def $w0 killed $w0 def $x0 +; SVE-NEXT: mov z0.d, x0 +; SVE-NEXT: mov z1.d, x1 +; SVE-NEXT: zip2 z2.d, z0.d, z1.d +; SVE-NEXT: zip1 z0.d, z0.d, z1.d +; SVE-NEXT: uzp1 z0.s, z0.s, z2.s +; SVE-NEXT: ret +; +; SME2-LABEL: interleave2_diff_nonconst_splat_nxv4i16: +; SME2: // %bb.0: +; SME2-NEXT: // kill: def $w1 killed $w1 def $x1 +; SME2-NEXT: // kill: def $w0 killed $w0 def $x0 +; SME2-NEXT: mov z0.d, x0 +; SME2-NEXT: mov z1.d, x1 +; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d +; SME2-NEXT: uzp1 z0.s, z0.s, z1.s +; SME2-NEXT: ret + %ins1 = insertelement <vscale x 2 x i16> poison, i16 %a, i32 0 + %splat1 = shufflevector <vscale x 2 x i16> %ins1, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer + %ins2 = insertelement <vscale x 2 x i16> poison, i16 %b, i32 0 + %splat2 = shufflevector <vscale x 2 x i16> %ins2, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer + %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %splat1, <vscale x 2 x i16> %splat2) + ret <vscale x 4 x i16> %retval +} + +define <vscale x 8 x i16> @interleave4_same_const_splat_nxv8i16() { +; CHECK-LABEL: interleave4_same_const_splat_nxv8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z0.h, #3 // =0x3 +; CHECK-NEXT: ret + %retval = call <vscale x 8 x i16> @llvm.vector.interleave4.nxv8i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3)) + ret <vscale x 8 x i16> %retval +} diff --git a/llvm/test/CodeGen/AArch64/sve-vscale-combine.ll b/llvm/test/CodeGen/AArch64/sve-vscale-combine.ll index 9306c20..7dcd56c 100644 --- a/llvm/test/CodeGen/AArch64/sve-vscale-combine.ll +++ b/llvm/test/CodeGen/AArch64/sve-vscale-combine.ll @@ -1,14 +1,14 @@ -; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve --asm-verbose=false < %s |FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mattr=+sve < %s | FileCheck %s -declare i32 @llvm.vscale.i32() -declare i64 @llvm.vscale.i64() +target triple = "aarch64-unknown-linux-gnu" ; Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)). define i64 @combine_add_vscale_i64() nounwind { ; CHECK-LABEL: combine_add_vscale_i64: -; CHECK-NOT: add -; CHECK-NEXT: cntd x0 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: cntd x0 +; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() %add = add i64 %vscale, %vscale ret i64 %add @@ -16,9 +16,10 @@ define i64 @combine_add_vscale_i64() nounwind { define i32 @combine_add_vscale_i32() nounwind { ; CHECK-LABEL: combine_add_vscale_i32: -; CHECK-NOT: add -; CHECK-NEXT: cntd x0 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: cntd x0 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret %vscale = call i32 @llvm.vscale.i32() %add = add i32 %vscale, %vscale ret i32 %add @@ -28,9 +29,9 @@ define i32 @combine_add_vscale_i32() nounwind { ; In this test, C0 = 1, C1 = 32. define i64 @combine_mul_vscale_i64() nounwind { ; CHECK-LABEL: combine_mul_vscale_i64: -; CHECK-NOT: mul -; CHECK-NEXT: rdvl x0, #2 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x0, #2 +; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() %mul = mul i64 %vscale, 32 ret i64 %mul @@ -38,9 +39,10 @@ define i64 @combine_mul_vscale_i64() nounwind { define i32 @combine_mul_vscale_i32() nounwind { ; CHECK-LABEL: combine_mul_vscale_i32: -; CHECK-NOT: mul -; CHECK-NEXT: rdvl x0, #3 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x0, #3 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret %vscale = call i32 @llvm.vscale.i32() %mul = mul i32 %vscale, 48 ret i32 %mul @@ -49,11 +51,11 @@ define i32 @combine_mul_vscale_i32() nounwind { ; Canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C)) define i64 @combine_sub_vscale_i64(i64 %in) nounwind { ; CHECK-LABEL: combine_sub_vscale_i64: -; CHECK-NOT: sub -; CHECK-NEXT: rdvl x8, #-1 -; CHECK-NEXT: asr x8, x8, #4 -; CHECK-NEXT: add x0, x0, x8 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x8, #-1 +; CHECK-NEXT: asr x8, x8, #4 +; CHECK-NEXT: add x0, x0, x8 +; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() %sub = sub i64 %in, %vscale ret i64 %sub @@ -61,11 +63,11 @@ define i64 @combine_sub_vscale_i64(i64 %in) nounwind { define i32 @combine_sub_vscale_i32(i32 %in) nounwind { ; CHECK-LABEL: combine_sub_vscale_i32: -; CHECK-NOT: sub -; CHECK-NEXT: rdvl x8, #-1 -; CHECK-NEXT: asr x8, x8, #4 -; CHECK-NEXT: add w0, w0, w8 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x8, #-1 +; CHECK-NEXT: asr x8, x8, #4 +; CHECK-NEXT: add w0, w0, w8 +; CHECK-NEXT: ret %vscale = call i32 @llvm.vscale.i32() %sub = sub i32 %in, %vscale ret i32 %sub @@ -75,12 +77,13 @@ define i32 @combine_sub_vscale_i32(i32 %in) nounwind { ; (sub X, (vscale * C)) to (add X, (vscale * -C)) define i64 @multiple_uses_sub_vscale_i64(i64 %x, i64 %y) nounwind { ; CHECK-LABEL: multiple_uses_sub_vscale_i64: -; CHECK-NEXT: rdvl x8, #1 -; CHECK-NEXT: lsr x8, x8, #4 -; CHECK-NEXT: sub x9, x0, x8 -; CHECK-NEXT: add x8, x1, x8 -; CHECK-NEXT: mul x0, x9, x8 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: lsr x8, x8, #4 +; CHECK-NEXT: sub x9, x0, x8 +; CHECK-NEXT: add x8, x1, x8 +; CHECK-NEXT: mul x0, x9, x8 +; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() %sub = sub i64 %x, %vscale %add = add i64 %y, %vscale @@ -95,9 +98,9 @@ define i64 @multiple_uses_sub_vscale_i64(i64 %x, i64 %y) nounwind { ; Hence, the immediate for RDVL is #1. define i64 @combine_shl_vscale_i64() nounwind { ; CHECK-LABEL: combine_shl_vscale_i64: -; CHECK-NOT: shl -; CHECK-NEXT: rdvl x0, #1 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x0, #1 +; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() %shl = shl i64 %vscale, 4 ret i64 %shl @@ -105,10 +108,38 @@ define i64 @combine_shl_vscale_i64() nounwind { define i32 @combine_shl_vscale_i32() nounwind { ; CHECK-LABEL: combine_shl_vscale_i32: -; CHECK-NOT: shl -; CHECK-NEXT: rdvl x0, #1 -; CHECK-NEXT: ret +; CHECK: // %bb.0: +; CHECK-NEXT: rdvl x0, #1 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret %vscale = call i32 @llvm.vscale.i32() %shl = shl i32 %vscale, 4 ret i32 %shl } + +define i64 @combine_shl_mul_vscale(i64 %a) nounwind { +; CHECK-LABEL: combine_shl_mul_vscale: +; CHECK: // %bb.0: +; CHECK-NEXT: cnth x8 +; CHECK-NEXT: mul x0, x0, x8 +; CHECK-NEXT: ret + %vscale = tail call i64 @llvm.vscale.i64() + %mul = mul i64 %a, %vscale + %shl = shl i64 %mul, 3 + ret i64 %shl +} + +define i64 @combine_shl_mul_vscale_commuted(i64 %a) nounwind { +; CHECK-LABEL: combine_shl_mul_vscale_commuted: +; CHECK: // %bb.0: +; CHECK-NEXT: cnth x8 +; CHECK-NEXT: mul x0, x0, x8 +; CHECK-NEXT: ret + %vscale = tail call i64 @llvm.vscale.i64() + %mul = mul i64 %vscale, %a + %shl = shl i64 %mul, 3 + ret i64 %shl +} + +declare i32 @llvm.vscale.i32() +declare i64 @llvm.vscale.i64() diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll index 290a473..74d1165 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll @@ -1907,11 +1907,8 @@ define i32 @test_udot_v8i8(<8 x i8> %a, <8 x i8> %b) { ; ; CHECK-GI-BASE-LABEL: test_udot_v8i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-GI-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h -; CHECK-GI-BASE-NEXT: addv s0, v2.4s +; CHECK-GI-BASE-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h ; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret ; @@ -1952,17 +1949,13 @@ define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; ; CHECK-GI-BASE-LABEL: test_udot_v16i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: umull v4.4s, v3.4h, v2.4h -; CHECK-GI-BASE-NEXT: umull v5.4s, v1.4h, v0.4h -; CHECK-GI-BASE-NEXT: umlal2 v4.4s, v3.8h, v2.8h -; CHECK-GI-BASE-NEXT: umlal2 v5.4s, v1.8h, v0.8h -; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v5.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: umull v2.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: umull2 v0.8h, v1.16b, v0.16b +; CHECK-GI-BASE-NEXT: uaddlv s1, v2.8h +; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h +; CHECK-GI-BASE-NEXT: fmov w8, s1 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 ; CHECK-GI-BASE-NEXT: ret ; ; CHECK-GI-DOT-LABEL: test_udot_v16i8: @@ -2018,36 +2011,21 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) { ; ; CHECK-GI-BASE-LABEL: test_udot_v24i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: fmov s0, wzr -; CHECK-GI-BASE-NEXT: fmov s1, wzr -; CHECK-GI-BASE-NEXT: ldr q2, [x0] -; CHECK-GI-BASE-NEXT: ldr d3, [x0, #16] -; CHECK-GI-BASE-NEXT: ldr q4, [x1] -; CHECK-GI-BASE-NEXT: ldr d5, [x1, #16] -; CHECK-GI-BASE-NEXT: ushll v6.8h, v2.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr -; CHECK-GI-BASE-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v7.8h, v4.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v4.8h, v4.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v5.8h, v5.8b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr -; CHECK-GI-BASE-NEXT: umull v16.4s, v7.4h, v6.4h -; CHECK-GI-BASE-NEXT: umull v17.4s, v4.4h, v2.4h -; CHECK-GI-BASE-NEXT: umull v18.4s, v5.4h, v3.4h -; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr -; CHECK-GI-BASE-NEXT: umlal2 v16.4s, v7.8h, v6.8h -; CHECK-GI-BASE-NEXT: umlal2 v17.4s, v4.8h, v2.8h -; CHECK-GI-BASE-NEXT: umlal2 v18.4s, v5.8h, v3.8h -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v16.4s, v17.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v18.4s, v0.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ldr q0, [x0] +; CHECK-GI-BASE-NEXT: ldr q1, [x1] +; CHECK-GI-BASE-NEXT: ldr d2, [x0, #16] +; CHECK-GI-BASE-NEXT: ldr d3, [x1, #16] +; CHECK-GI-BASE-NEXT: umull v4.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: umull2 v0.8h, v1.16b, v0.16b +; CHECK-GI-BASE-NEXT: umull v1.8h, v3.8b, v2.8b +; CHECK-GI-BASE-NEXT: uaddlv s2, v4.8h +; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h +; CHECK-GI-BASE-NEXT: uaddlv s1, v1.8h +; CHECK-GI-BASE-NEXT: fmov w8, s2 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: add w8, w8, w9 +; CHECK-GI-BASE-NEXT: fmov w9, s1 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 ; CHECK-GI-BASE-NEXT: ret ; ; CHECK-GI-DOT-LABEL: test_udot_v24i8: @@ -2118,61 +2096,33 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) { ; ; CHECK-GI-BASE-LABEL: test_udot_v48i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: fmov s0, wzr -; CHECK-GI-BASE-NEXT: fmov s2, wzr -; CHECK-GI-BASE-NEXT: ldr q16, [x0, #32] -; CHECK-GI-BASE-NEXT: fmov s1, wzr -; CHECK-GI-BASE-NEXT: fmov s3, wzr -; CHECK-GI-BASE-NEXT: ldr q19, [x1, #32] -; CHECK-GI-BASE-NEXT: ldp q5, q7, [x1] -; CHECK-GI-BASE-NEXT: ushll v23.8h, v16.8b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr -; CHECK-GI-BASE-NEXT: mov v2.s[1], wzr -; CHECK-GI-BASE-NEXT: ushll v20.8h, v19.8b, #0 -; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr -; CHECK-GI-BASE-NEXT: mov v3.s[1], wzr -; CHECK-GI-BASE-NEXT: ushll2 v19.8h, v19.16b, #0 -; CHECK-GI-BASE-NEXT: ldp q18, q17, [x0] -; CHECK-GI-BASE-NEXT: ushll v4.8h, v5.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v5.8h, v5.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v6.8h, v7.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v7.8h, v7.16b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v16.8h, v16.16b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr -; CHECK-GI-BASE-NEXT: mov v2.s[2], wzr -; CHECK-GI-BASE-NEXT: ushll v21.8h, v18.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v18.8h, v18.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v22.8h, v17.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v17.8h, v17.16b, #0 -; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr -; CHECK-GI-BASE-NEXT: mov v3.s[2], wzr -; CHECK-GI-BASE-NEXT: umull v28.4s, v20.4h, v23.4h -; CHECK-GI-BASE-NEXT: umull v29.4s, v19.4h, v16.4h -; CHECK-GI-BASE-NEXT: umull v24.4s, v4.4h, v21.4h -; CHECK-GI-BASE-NEXT: umull v25.4s, v5.4h, v18.4h -; CHECK-GI-BASE-NEXT: umull v26.4s, v6.4h, v22.4h -; CHECK-GI-BASE-NEXT: umull v27.4s, v7.4h, v17.4h -; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v2.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v3.s[3], wzr -; CHECK-GI-BASE-NEXT: umlal2 v28.4s, v20.8h, v23.8h -; CHECK-GI-BASE-NEXT: umlal2 v29.4s, v19.8h, v16.8h -; CHECK-GI-BASE-NEXT: umlal2 v24.4s, v4.8h, v21.8h -; CHECK-GI-BASE-NEXT: umlal2 v25.4s, v5.8h, v18.8h -; CHECK-GI-BASE-NEXT: umlal2 v26.4s, v6.8h, v22.8h -; CHECK-GI-BASE-NEXT: umlal2 v27.4s, v7.8h, v17.8h -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v1.4s, v3.4s -; CHECK-GI-BASE-NEXT: add v4.4s, v28.4s, v29.4s -; CHECK-GI-BASE-NEXT: add v2.4s, v24.4s, v25.4s -; CHECK-GI-BASE-NEXT: add v3.4s, v26.4s, v27.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v2.4s, v3.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v0.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ldp q0, q1, [x0] +; CHECK-GI-BASE-NEXT: ldr q3, [x0, #32] +; CHECK-GI-BASE-NEXT: ldp q2, q4, [x1] +; CHECK-GI-BASE-NEXT: ldr q5, [x1, #32] +; CHECK-GI-BASE-NEXT: umull v7.8h, v5.8b, v3.8b +; CHECK-GI-BASE-NEXT: umull2 v3.8h, v5.16b, v3.16b +; CHECK-GI-BASE-NEXT: umull v6.8h, v2.8b, v0.8b +; CHECK-GI-BASE-NEXT: umull2 v0.8h, v2.16b, v0.16b +; CHECK-GI-BASE-NEXT: umull2 v2.8h, v4.16b, v1.16b +; CHECK-GI-BASE-NEXT: umull v1.8h, v4.8b, v1.8b +; CHECK-GI-BASE-NEXT: uaddlv s5, v7.8h +; CHECK-GI-BASE-NEXT: uaddlv s3, v3.8h +; CHECK-GI-BASE-NEXT: uaddlv s4, v6.8h +; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h +; CHECK-GI-BASE-NEXT: uaddlv s2, v2.8h +; CHECK-GI-BASE-NEXT: uaddlv s1, v1.8h +; CHECK-GI-BASE-NEXT: fmov w11, s5 +; CHECK-GI-BASE-NEXT: fmov w8, s4 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: fmov w10, s2 +; CHECK-GI-BASE-NEXT: add w8, w8, w9 +; CHECK-GI-BASE-NEXT: fmov w9, s1 +; CHECK-GI-BASE-NEXT: add w10, w10, w11 +; CHECK-GI-BASE-NEXT: fmov w11, s3 +; CHECK-GI-BASE-NEXT: add w8, w8, w9 +; CHECK-GI-BASE-NEXT: add w9, w10, w11 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 ; CHECK-GI-BASE-NEXT: ret ; ; CHECK-GI-DOT-LABEL: test_udot_v48i8: @@ -2225,11 +2175,8 @@ define i32 @test_sdot_v8i8(<8 x i8> %a, <8 x i8> %b) { ; ; CHECK-GI-BASE-LABEL: test_sdot_v8i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: smull v2.4s, v1.4h, v0.4h -; CHECK-GI-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h -; CHECK-GI-BASE-NEXT: addv s0, v2.4s +; CHECK-GI-BASE-NEXT: smull v0.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h ; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret ; @@ -2270,17 +2217,13 @@ define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; ; CHECK-GI-BASE-LABEL: test_sdot_v16i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: smull v4.4s, v3.4h, v2.4h -; CHECK-GI-BASE-NEXT: smull v5.4s, v1.4h, v0.4h -; CHECK-GI-BASE-NEXT: smlal2 v4.4s, v3.8h, v2.8h -; CHECK-GI-BASE-NEXT: smlal2 v5.4s, v1.8h, v0.8h -; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v5.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: smull v2.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: smull2 v0.8h, v1.16b, v0.16b +; CHECK-GI-BASE-NEXT: saddlv s1, v2.8h +; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h +; CHECK-GI-BASE-NEXT: fmov w8, s1 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 ; CHECK-GI-BASE-NEXT: ret ; ; CHECK-GI-DOT-LABEL: test_sdot_v16i8: @@ -2336,36 +2279,21 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) { ; ; CHECK-GI-BASE-LABEL: test_sdot_v24i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: fmov s0, wzr -; CHECK-GI-BASE-NEXT: fmov s1, wzr -; CHECK-GI-BASE-NEXT: ldr q2, [x0] -; CHECK-GI-BASE-NEXT: ldr d3, [x0, #16] -; CHECK-GI-BASE-NEXT: ldr q4, [x1] -; CHECK-GI-BASE-NEXT: ldr d5, [x1, #16] -; CHECK-GI-BASE-NEXT: sshll v6.8h, v2.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v2.8h, v2.16b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr -; CHECK-GI-BASE-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v7.8h, v4.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v4.8h, v4.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v5.8h, v5.8b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr -; CHECK-GI-BASE-NEXT: smull v16.4s, v7.4h, v6.4h -; CHECK-GI-BASE-NEXT: smull v17.4s, v4.4h, v2.4h -; CHECK-GI-BASE-NEXT: smull v18.4s, v5.4h, v3.4h -; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr -; CHECK-GI-BASE-NEXT: smlal2 v16.4s, v7.8h, v6.8h -; CHECK-GI-BASE-NEXT: smlal2 v17.4s, v4.8h, v2.8h -; CHECK-GI-BASE-NEXT: smlal2 v18.4s, v5.8h, v3.8h -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v16.4s, v17.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v18.4s, v0.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ldr q0, [x0] +; CHECK-GI-BASE-NEXT: ldr q1, [x1] +; CHECK-GI-BASE-NEXT: ldr d2, [x0, #16] +; CHECK-GI-BASE-NEXT: ldr d3, [x1, #16] +; CHECK-GI-BASE-NEXT: smull v4.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: smull2 v0.8h, v1.16b, v0.16b +; CHECK-GI-BASE-NEXT: smull v1.8h, v3.8b, v2.8b +; CHECK-GI-BASE-NEXT: saddlv s2, v4.8h +; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h +; CHECK-GI-BASE-NEXT: saddlv s1, v1.8h +; CHECK-GI-BASE-NEXT: fmov w8, s2 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: add w8, w8, w9 +; CHECK-GI-BASE-NEXT: fmov w9, s1 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 ; CHECK-GI-BASE-NEXT: ret ; ; CHECK-GI-DOT-LABEL: test_sdot_v24i8: @@ -2436,61 +2364,33 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) { ; ; CHECK-GI-BASE-LABEL: test_sdot_v48i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: fmov s0, wzr -; CHECK-GI-BASE-NEXT: fmov s2, wzr -; CHECK-GI-BASE-NEXT: ldr q16, [x0, #32] -; CHECK-GI-BASE-NEXT: fmov s1, wzr -; CHECK-GI-BASE-NEXT: fmov s3, wzr -; CHECK-GI-BASE-NEXT: ldr q19, [x1, #32] -; CHECK-GI-BASE-NEXT: ldp q5, q7, [x1] -; CHECK-GI-BASE-NEXT: sshll v23.8h, v16.8b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[1], wzr -; CHECK-GI-BASE-NEXT: mov v2.s[1], wzr -; CHECK-GI-BASE-NEXT: sshll v20.8h, v19.8b, #0 -; CHECK-GI-BASE-NEXT: mov v1.s[1], wzr -; CHECK-GI-BASE-NEXT: mov v3.s[1], wzr -; CHECK-GI-BASE-NEXT: sshll2 v19.8h, v19.16b, #0 -; CHECK-GI-BASE-NEXT: ldp q18, q17, [x0] -; CHECK-GI-BASE-NEXT: sshll v4.8h, v5.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v5.8h, v5.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v6.8h, v7.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v7.8h, v7.16b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v16.8h, v16.16b, #0 -; CHECK-GI-BASE-NEXT: mov v0.s[2], wzr -; CHECK-GI-BASE-NEXT: mov v2.s[2], wzr -; CHECK-GI-BASE-NEXT: sshll v21.8h, v18.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v18.8h, v18.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v22.8h, v17.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v17.8h, v17.16b, #0 -; CHECK-GI-BASE-NEXT: mov v1.s[2], wzr -; CHECK-GI-BASE-NEXT: mov v3.s[2], wzr -; CHECK-GI-BASE-NEXT: smull v28.4s, v20.4h, v23.4h -; CHECK-GI-BASE-NEXT: smull v29.4s, v19.4h, v16.4h -; CHECK-GI-BASE-NEXT: smull v24.4s, v4.4h, v21.4h -; CHECK-GI-BASE-NEXT: smull v25.4s, v5.4h, v18.4h -; CHECK-GI-BASE-NEXT: smull v26.4s, v6.4h, v22.4h -; CHECK-GI-BASE-NEXT: smull v27.4s, v7.4h, v17.4h -; CHECK-GI-BASE-NEXT: mov v0.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v2.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v1.s[3], wzr -; CHECK-GI-BASE-NEXT: mov v3.s[3], wzr -; CHECK-GI-BASE-NEXT: smlal2 v28.4s, v20.8h, v23.8h -; CHECK-GI-BASE-NEXT: smlal2 v29.4s, v19.8h, v16.8h -; CHECK-GI-BASE-NEXT: smlal2 v24.4s, v4.8h, v21.8h -; CHECK-GI-BASE-NEXT: smlal2 v25.4s, v5.8h, v18.8h -; CHECK-GI-BASE-NEXT: smlal2 v26.4s, v6.8h, v22.8h -; CHECK-GI-BASE-NEXT: smlal2 v27.4s, v7.8h, v17.8h -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v1.4s, v3.4s -; CHECK-GI-BASE-NEXT: add v4.4s, v28.4s, v29.4s -; CHECK-GI-BASE-NEXT: add v2.4s, v24.4s, v25.4s -; CHECK-GI-BASE-NEXT: add v3.4s, v26.4s, v27.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v2.4s, v3.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v0.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ldp q0, q1, [x0] +; CHECK-GI-BASE-NEXT: ldr q3, [x0, #32] +; CHECK-GI-BASE-NEXT: ldp q2, q4, [x1] +; CHECK-GI-BASE-NEXT: ldr q5, [x1, #32] +; CHECK-GI-BASE-NEXT: smull v7.8h, v5.8b, v3.8b +; CHECK-GI-BASE-NEXT: smull2 v3.8h, v5.16b, v3.16b +; CHECK-GI-BASE-NEXT: smull v6.8h, v2.8b, v0.8b +; CHECK-GI-BASE-NEXT: smull2 v0.8h, v2.16b, v0.16b +; CHECK-GI-BASE-NEXT: smull2 v2.8h, v4.16b, v1.16b +; CHECK-GI-BASE-NEXT: smull v1.8h, v4.8b, v1.8b +; CHECK-GI-BASE-NEXT: saddlv s5, v7.8h +; CHECK-GI-BASE-NEXT: saddlv s3, v3.8h +; CHECK-GI-BASE-NEXT: saddlv s4, v6.8h +; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h +; CHECK-GI-BASE-NEXT: saddlv s2, v2.8h +; CHECK-GI-BASE-NEXT: saddlv s1, v1.8h +; CHECK-GI-BASE-NEXT: fmov w11, s5 +; CHECK-GI-BASE-NEXT: fmov w8, s4 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: fmov w10, s2 +; CHECK-GI-BASE-NEXT: add w8, w8, w9 +; CHECK-GI-BASE-NEXT: fmov w9, s1 +; CHECK-GI-BASE-NEXT: add w10, w10, w11 +; CHECK-GI-BASE-NEXT: fmov w11, s3 +; CHECK-GI-BASE-NEXT: add w8, w8, w9 +; CHECK-GI-BASE-NEXT: add w9, w10, w11 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 ; CHECK-GI-BASE-NEXT: ret ; ; CHECK-GI-DOT-LABEL: test_sdot_v48i8: @@ -2549,18 +2449,27 @@ define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-DOT-NEXT: add w0, w8, w9 ; CHECK-SD-DOT-NEXT: ret ; -; CHECK-GI-LABEL: test_udot_v8i8_multi_use: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-GI-NEXT: mov v3.16b, v2.16b -; CHECK-GI-NEXT: fmov w8, s2 -; CHECK-GI-NEXT: umlal2 v3.4s, v1.8h, v0.8h -; CHECK-GI-NEXT: addv s0, v3.4s -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: add w0, w9, w8 -; CHECK-GI-NEXT: ret +; CHECK-GI-BASE-LABEL: test_udot_v8i8_multi_use: +; CHECK-GI-BASE: // %bb.0: // %entry +; CHECK-GI-BASE-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: uaddlv s1, v0.8h +; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-BASE-NEXT: fmov w9, s0 +; CHECK-GI-BASE-NEXT: fmov w8, s1 +; CHECK-GI-BASE-NEXT: add w0, w8, w9 +; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: test_udot_v8i8_multi_use: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: umull v3.8h, v1.8b, v0.8b +; CHECK-GI-DOT-NEXT: udot v2.2s, v1.8b, v0.8b +; CHECK-GI-DOT-NEXT: ushll v0.4s, v3.4h, #0 +; CHECK-GI-DOT-NEXT: fmov w9, s0 +; CHECK-GI-DOT-NEXT: addp v1.2s, v2.2s, v2.2s +; CHECK-GI-DOT-NEXT: fmov w8, s1 +; CHECK-GI-DOT-NEXT: add w0, w8, w9 +; CHECK-GI-DOT-NEXT: ret entry: %0 = zext <8 x i8> %a to <8 x i32> %1 = zext <8 x i8> %b to <8 x i32> |