diff options
| author | Tuan Chuong Goh <chuong.goh@arm.com> | 2024-01-15 17:35:48 +0000 |
|---|---|---|
| committer | Tuan Chuong Goh <chuong.goh@arm.com> | 2024-01-15 17:54:52 +0000 |
| commit | 22c24be37c806e9295b05a34546b61a3164be267 (patch) | |
| tree | 44a92612e09e344a0f5145ed1415b04b86286394 | |
| parent | 2e08e821b7ea5bf7c0fe0775feb94a7fdb5204c7 (diff) | |
| download | llvm-22c24be37c806e9295b05a34546b61a3164be267.zip llvm-22c24be37c806e9295b05a34546b61a3164be267.tar.gz llvm-22c24be37c806e9295b05a34546b61a3164be267.tar.bz2 | |
[AArch64][GlobalISel] Pre-commit for Combine vecreduce(ext) to {U/S}ADDLV
| -rw-r--r-- | llvm/test/CodeGen/AArch64/vecreduce-add.ll | 6538 |
1 files changed, 2476 insertions, 4062 deletions
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll index 5fa28f7..32f5bfc 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BASE,CHECK-SD-BASE -; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-SD-DOT -; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BASE,CHECK-GI-BASE -; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - -mattr=+dotprod 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-GI-DOT +; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-BASE +; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-DOT +; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-BASE +; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 %s -o - -mattr=+dotprod 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-DOT ; CHECK-GI-BASE: warning: Instruction selection used fallback path for full @@ -51,33 +51,19 @@ entry: } define i64 @add_v4i32_v4i64_zext(<4 x i32> %x) { -; CHECK-SD-BASE-LABEL: add_v4i32_v4i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i32_v4i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i32_v4i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i32_v4i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i32_v4i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i32_v4i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i32> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -85,33 +71,19 @@ entry: } define i64 @add_v4i32_v4i64_sext(<4 x i32> %x) { -; CHECK-SD-BASE-LABEL: add_v4i32_v4i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i32_v4i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i32_v4i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i32_v4i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i32_v4i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i32_v4i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i32> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -145,33 +117,19 @@ entry: } define i32 @add_v8i16_v8i32_zext(<8 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i32_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlv s0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i32_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlv s0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i32_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i32_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w0, s0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i32_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlv s0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i32_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: uaddw2 v0.4s, v1.4s, v0.8h +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w0, s0 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i16> %x to <8 x i32> %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) @@ -179,33 +137,19 @@ entry: } define i32 @add_v8i16_v8i32_sext(<8 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i32_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlv s0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i32_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlv s0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i32_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i32_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w0, s0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i32_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlv s0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i32_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: saddw2 v0.4s, v1.4s, v0.8h +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w0, s0 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i16> %x to <8 x i32> %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) @@ -239,84 +183,47 @@ entry: } define zeroext i16 @add_v8i16_v8i16(<8 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i16: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i16: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i16: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: uxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i16: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: uxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: uxth w0, w8 +; CHECK-GI-NEXT: ret entry: %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x) ret i16 %z } define i64 @add_v8i16_v8i64_zext(<8 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i16> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -324,53 +231,29 @@ entry: } define i64 @add_v8i16_v8i64_sext(<8 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i16> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -378,37 +261,21 @@ entry: } define i64 @add_v4i16_v4i64_zext(<4 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v4i16_v4i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i16_v4i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i16_v4i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i16_v4i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i16_v4i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i16_v4i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i16> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -416,37 +283,21 @@ entry: } define i64 @add_v4i16_v4i64_sext(<4 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v4i16_v4i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i16_v4i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i16_v4i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i16_v4i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i16_v4i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i16_v4i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i16> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -454,41 +305,23 @@ entry: } define i64 @add_v2i16_v2i64_zext(<2 x i16> %x) { -; CHECK-SD-BASE-LABEL: add_v2i16_v2i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: movi d1, #0x00ffff0000ffff -; CHECK-SD-BASE-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v2i16_v2i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: movi d1, #0x00ffff0000ffff -; CHECK-SD-DOT-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v2i16_v2i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x0000000000ffff -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v2i16_v2i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x0000000000ffff -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v2i16_v2i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi d1, #0x00ffff0000ffff +; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v2i16_v2i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v1.2d, #0x0000000000ffff +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i16> %x to <2 x i64> %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -522,14 +355,14 @@ define i32 @add_v16i8_v16i32_zext(<16 x i8> %x) { ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-DOT-LABEL: add_v16i8_v16i32_zext: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v1.16b, #1 -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v1.16b -; CHECK-DOT-NEXT: addv s0, v2.4s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_zext: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v1.16b, #1 +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: udot v2.4s, v0.16b, v1.16b +; CHECK-SD-DOT-NEXT: addv s0, v2.4s +; CHECK-SD-DOT-NEXT: fmov w0, s0 +; CHECK-SD-DOT-NEXT: ret ; ; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_zext: ; CHECK-GI-BASE: // %bb.0: // %entry @@ -543,6 +376,15 @@ define i32 @add_v16i8_v16i32_zext(<16 x i8> %x) { ; CHECK-GI-BASE-NEXT: addv s0, v0.4s ; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_zext: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v1.16b, #1 +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: udot v2.4s, v0.16b, v1.16b +; CHECK-GI-DOT-NEXT: addv s0, v2.4s +; CHECK-GI-DOT-NEXT: fmov w0, s0 +; CHECK-GI-DOT-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i32> %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) @@ -561,14 +403,14 @@ define i32 @add_v16i8_v16i32_sext(<16 x i8> %x) { ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-DOT-LABEL: add_v16i8_v16i32_sext: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v1.16b, #1 -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b -; CHECK-DOT-NEXT: addv s0, v2.4s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_sext: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v1.16b, #1 +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b +; CHECK-SD-DOT-NEXT: addv s0, v2.4s +; CHECK-SD-DOT-NEXT: fmov w0, s0 +; CHECK-SD-DOT-NEXT: ret ; ; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_sext: ; CHECK-GI-BASE: // %bb.0: // %entry @@ -582,6 +424,15 @@ define i32 @add_v16i8_v16i32_sext(<16 x i8> %x) { ; CHECK-GI-BASE-NEXT: addv s0, v0.4s ; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_sext: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v1.16b, #1 +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b +; CHECK-GI-DOT-NEXT: addv s0, v2.4s +; CHECK-GI-DOT-NEXT: fmov w0, s0 +; CHECK-GI-DOT-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i32> %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) @@ -669,39 +520,22 @@ entry: } define i32 @add_v4i8_v4i32_zext(<4 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v4i8_v4i32_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i8_v4i32_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i8_v4i32_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x0000ff000000ff -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i8_v4i32_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x0000ff000000ff -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w0, s0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i8_v4i32_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i8_v4i32_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v1.2d, #0x0000ff000000ff +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w0, s0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i8> %x to <4 x i32> %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) @@ -724,37 +558,21 @@ entry: } define zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i16_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlp v0.8h, v0.16b -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i16_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlp v0.8h, v0.16b -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i16_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: uxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i16_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: uxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i16_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlp v0.8h, v0.16b +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i16_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: uaddw2 v0.8h, v1.8h, v0.16b +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: uxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i16> %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) @@ -762,37 +580,21 @@ entry: } define signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i16_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlp v0.8h, v0.16b -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: smov w0, v0.h[0] -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i16_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlp v0.8h, v0.16b -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: smov w0, v0.h[0] -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i16_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: sxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i16_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: sxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i16_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlp v0.8h, v0.16b +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: smov w0, v0.h[0] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i16_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: saddw2 v0.8h, v1.8h, v0.16b +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: sxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i16> %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) @@ -800,35 +602,20 @@ entry: } define zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i16_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i16_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i16_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: uxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i16_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: uxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i16_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i16_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: uxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i8> %x to <8 x i16> %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) @@ -836,35 +623,20 @@ entry: } define signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i16_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: smov w0, v0.h[0] -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i16_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: smov w0, v0.h[0] -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i16_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: sxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i16_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: sxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i16_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: smov w0, v0.h[0] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i16_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: sxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i8> %x to <8 x i16> %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) @@ -872,120 +644,65 @@ entry: } define zeroext i8 @add_v16i8_v16i8(<16 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i8: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: addv b0, v0.16b -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i8: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: addv b0, v0.16b -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i8: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv b0, v0.16b -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: uxtb w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i8: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv b0, v0.16b -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: uxtb w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: addv b0, v0.16b +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv b0, v0.16b +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: uxtb w0, w8 +; CHECK-GI-NEXT: ret entry: %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x) ret i8 %z } define i64 @add_v16i8_v16i64_zext(<16 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v2.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v3.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-BASE-NEXT: uaddl v2.2d, v3.2s, v2.2s -; CHECK-SD-BASE-NEXT: uaddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll2 v1.8h, v0.16b, #0 -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll2 v2.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v3.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-DOT-NEXT: uaddl v2.2d, v3.2s, v2.2s -; CHECK-SD-DOT-NEXT: uaddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v2.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v3.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v5.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v7.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-BASE-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-DOT-NEXT: ushll v2.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v3.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v5.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v7.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-DOT-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll2 v1.8h, v0.16b, #0 +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: ushll2 v2.4s, v1.8h, #0 +; CHECK-SD-NEXT: ushll2 v3.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddl2 v4.2d, v3.4s, v2.4s +; CHECK-SD-NEXT: uaddl v2.2d, v3.2s, v2.2s +; CHECK-SD-NEXT: uaddl2 v5.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ushll v2.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: ushll v5.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll v7.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v5.2d, v1.4s +; CHECK-GI-NEXT: uaddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v7.2d, v0.4s +; CHECK-GI-NEXT: add v1.2d, v2.2d, v1.2d +; CHECK-GI-NEXT: add v0.2d, v3.2d, v0.2d +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i64> %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) @@ -993,89 +710,47 @@ entry: } define i64 @add_v16i8_v16i64_sext(<16 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v2.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v3.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-BASE-NEXT: saddl v2.2d, v3.2s, v2.2s -; CHECK-SD-BASE-NEXT: saddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll2 v1.8h, v0.16b, #0 -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: sshll2 v2.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v3.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-DOT-NEXT: saddl v2.2d, v3.2s, v2.2s -; CHECK-SD-DOT-NEXT: saddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v2.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v3.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v5.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v7.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-BASE-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-DOT-NEXT: sshll v2.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v3.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v5.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v7.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-DOT-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll2 v1.8h, v0.16b, #0 +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll2 v2.4s, v1.8h, #0 +; CHECK-SD-NEXT: sshll2 v3.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddl2 v4.2d, v3.4s, v2.4s +; CHECK-SD-NEXT: saddl v2.2d, v3.2s, v2.2s +; CHECK-SD-NEXT: saddl2 v5.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v2.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: sshll v5.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: sshll v7.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v5.2d, v1.4s +; CHECK-GI-NEXT: saddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v7.2d, v0.4s +; CHECK-GI-NEXT: add v1.2d, v2.2d, v1.2d +; CHECK-GI-NEXT: add v0.2d, v3.2d, v0.2d +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i64> %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) @@ -1083,57 +758,31 @@ entry: } define i64 @add_v8i8_v8i64_zext(<8 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i8> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -1141,57 +790,31 @@ entry: } define i64 @add_v8i8_v8i64_sext(<8 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i8> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -1199,47 +822,26 @@ entry: } define i64 @add_v4i8_v4i64_zext(<4 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v4i8_v4i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i8_v4i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i8_v4i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-BASE-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-GI-BASE-NEXT: and v2.16b, v2.16b, v1.16b -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i8_v4i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-DOT-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-GI-DOT-NEXT: and v2.16b, v2.16b, v1.16b -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i8_v4i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i8_v4i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: movi v1.2d, #0x000000000000ff +; CHECK-GI-NEXT: ushll v2.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll2 v0.2d, v0.4s, #0 +; CHECK-GI-NEXT: and v2.16b, v2.16b, v1.16b +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i8> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -1247,57 +849,31 @@ entry: } define i64 @add_v4i8_v4i64_sext(<4 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v4i8_v4i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-SD-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i8_v4i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-SD-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i8_v4i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.2d, v0.4s, #0 -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: addp d0, v1.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i8_v4i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.2d, v0.4s, #0 -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: addp d0, v1.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i8_v4i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll v1.2d, v0.2s, #0 +; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #0 +; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #56 +; CHECK-SD-NEXT: ssra v1.2d, v0.2d, #56 +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i8_v4i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.2d, v0.4s, #0 +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: ssra v1.2d, v0.2d, #56 +; CHECK-GI-NEXT: addp d0, v1.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i8> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -1305,41 +881,23 @@ entry: } define i64 @add_v2i8_v2i64_zext(<2 x i8> %x) { -; CHECK-SD-BASE-LABEL: add_v2i8_v2i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: movi d1, #0x0000ff000000ff -; CHECK-SD-BASE-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v2i8_v2i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: movi d1, #0x0000ff000000ff -; CHECK-SD-DOT-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v2i8_v2i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x0, d0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v2i8_v2i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x0, d0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v2i8_v2i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi d1, #0x0000ff000000ff +; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v2i8_v2i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v1.2d, #0x000000000000ff +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x0, d0 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i8> %x to <2 x i64> %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -1386,37 +944,21 @@ entry: } define i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v4i32_v4i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i32_v4i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i32_v4i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i32_v4i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i32_v4i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i32_v4i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i32> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -1425,37 +967,21 @@ entry: } define i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v4i32_v4i64_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i32_v4i64_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i32_v4i64_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i32_v4i64_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i32_v4i64_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i32_v4i64_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i32> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -1494,37 +1020,21 @@ entry: } define i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, i32 %a) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i32_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlv s0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w0, w8, w0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i32_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlv s0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w0, w8, w0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i32_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w0, w8, w0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i32_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w0, w8, w0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i32_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlv s0, v0.8h +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i32_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: uaddw2 v0.4s, v1.4s, v0.8h +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w0 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i16> %x to <8 x i32> %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) @@ -1533,37 +1043,21 @@ entry: } define i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, i32 %a) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i32_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlv s0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w0, w8, w0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i32_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlv s0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w0, w8, w0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i32_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w0, w8, w0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i32_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.4s, v1.4s, v0.8h -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w0, w8, w0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i32_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlv s0, v0.8h +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i32_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: saddw2 v0.4s, v1.4s, v0.8h +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w0 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i16> %x to <8 x i32> %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) @@ -1602,37 +1096,21 @@ entry: } define zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, i16 %a) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i16_acc: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w8, w8, w0 -; CHECK-SD-BASE-NEXT: and w0, w8, #0xffff -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i16_acc: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w8, w8, w0 -; CHECK-SD-DOT-NEXT: and w0, w8, #0xffff -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i16_acc: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w8, w0, w8, uxth -; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i16_acc: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w8, w0, w8, uxth -; CHECK-GI-DOT-NEXT: and w0, w8, #0xffff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i16_acc: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w8, w8, w0 +; CHECK-SD-NEXT: and w0, w8, #0xffff +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i16_acc: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w8, w0, w8, uxth +; CHECK-GI-NEXT: and w0, w8, #0xffff +; CHECK-GI-NEXT: ret entry: %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x) %r = add i16 %z, %a @@ -1640,57 +1118,31 @@ entry: } define i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i16> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -1699,57 +1151,31 @@ entry: } define i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v8i16_v8i64_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i16_v8i64_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i16_v8i64_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i16_v8i64_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i16_v8i64_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i16_v8i64_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i16> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -1758,41 +1184,23 @@ entry: } define i64 @add_v4i16_v4i64_acc_zext(<4 x i16> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v4i16_v4i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i16_v4i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i16_v4i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i16_v4i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i16_v4i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i16_v4i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i16> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -1801,41 +1209,23 @@ entry: } define i64 @add_v4i16_v4i64_acc_sext(<4 x i16> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v4i16_v4i64_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i16_v4i64_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i16_v4i64_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i16_v4i64_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll v1.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v1.2d, v0.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i16_v4i64_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i16_v4i64_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v1.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v0.2d, v1.2d, v0.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i16> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -1844,45 +1234,25 @@ entry: } define i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v2i16_v2i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: movi d1, #0x00ffff0000ffff -; CHECK-SD-BASE-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v2i16_v2i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: movi d1, #0x00ffff0000ffff -; CHECK-SD-DOT-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v2i16_v2i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x0000000000ffff -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v2i16_v2i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x0000000000ffff -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v2i16_v2i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi d1, #0x00ffff0000ffff +; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v2i16_v2i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v1.2d, #0x0000000000ffff +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i16> %x to <2 x i64> %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -1920,15 +1290,15 @@ define i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, i32 %a) { ; CHECK-SD-BASE-NEXT: add w0, w8, w0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-DOT-LABEL: add_v16i8_v16i32_acc_zext: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v1.16b, #1 -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v1.16b -; CHECK-DOT-NEXT: addv s0, v2.4s -; CHECK-DOT-NEXT: fmov w8, s0 -; CHECK-DOT-NEXT: add w0, w8, w0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_acc_zext: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v1.16b, #1 +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: udot v2.4s, v0.16b, v1.16b +; CHECK-SD-DOT-NEXT: addv s0, v2.4s +; CHECK-SD-DOT-NEXT: fmov w8, s0 +; CHECK-SD-DOT-NEXT: add w0, w8, w0 +; CHECK-SD-DOT-NEXT: ret ; ; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_acc_zext: ; CHECK-GI-BASE: // %bb.0: // %entry @@ -1943,6 +1313,16 @@ define i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, i32 %a) { ; CHECK-GI-BASE-NEXT: fmov w8, s0 ; CHECK-GI-BASE-NEXT: add w0, w8, w0 ; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_acc_zext: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v1.16b, #1 +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: udot v2.4s, v0.16b, v1.16b +; CHECK-GI-DOT-NEXT: addv s0, v2.4s +; CHECK-GI-DOT-NEXT: fmov w8, s0 +; CHECK-GI-DOT-NEXT: add w0, w8, w0 +; CHECK-GI-DOT-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i32> %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) @@ -1963,15 +1343,15 @@ define i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, i32 %a) { ; CHECK-SD-BASE-NEXT: add w0, w8, w0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-DOT-LABEL: add_v16i8_v16i32_acc_sext: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v1.16b, #1 -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b -; CHECK-DOT-NEXT: addv s0, v2.4s -; CHECK-DOT-NEXT: fmov w8, s0 -; CHECK-DOT-NEXT: add w0, w8, w0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_acc_sext: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v1.16b, #1 +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b +; CHECK-SD-DOT-NEXT: addv s0, v2.4s +; CHECK-SD-DOT-NEXT: fmov w8, s0 +; CHECK-SD-DOT-NEXT: add w0, w8, w0 +; CHECK-SD-DOT-NEXT: ret ; ; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_acc_sext: ; CHECK-GI-BASE: // %bb.0: // %entry @@ -1986,6 +1366,16 @@ define i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, i32 %a) { ; CHECK-GI-BASE-NEXT: fmov w8, s0 ; CHECK-GI-BASE-NEXT: add w0, w8, w0 ; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_acc_sext: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v1.16b, #1 +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b +; CHECK-GI-DOT-NEXT: addv s0, v2.4s +; CHECK-GI-DOT-NEXT: fmov w8, s0 +; CHECK-GI-DOT-NEXT: add w0, w8, w0 +; CHECK-GI-DOT-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i32> %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx) @@ -2084,43 +1474,24 @@ entry: } define i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, i32 %a) { -; CHECK-SD-BASE-LABEL: add_v4i8_v4i32_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w0, w8, w0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i8_v4i32_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w0, w8, w0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i8_v4i32_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x0000ff000000ff -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w0, w8, w0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i8_v4i32_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x0000ff000000ff -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w0, w8, w0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i8_v4i32_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i8_v4i32_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v1.2d, #0x0000ff000000ff +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i8> %x to <4 x i32> %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) @@ -2146,41 +1517,23 @@ entry: } define zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, i16 %a) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i16_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlv h0, v0.16b -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w8, w8, w0 -; CHECK-SD-BASE-NEXT: and w0, w8, #0xffff -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i16_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlv h0, v0.16b -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w8, w8, w0 -; CHECK-SD-DOT-NEXT: and w0, w8, #0xffff -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i16_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w8, w0, w8, uxth -; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i16_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w8, w0, w8, uxth -; CHECK-GI-DOT-NEXT: and w0, w8, #0xffff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i16_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlv h0, v0.16b +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w8, w8, w0 +; CHECK-SD-NEXT: and w0, w8, #0xffff +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i16_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: uaddw2 v0.8h, v1.8h, v0.16b +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w8, w0, w8, uxth +; CHECK-GI-NEXT: and w0, w8, #0xffff +; CHECK-GI-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i16> %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) @@ -2189,41 +1542,23 @@ entry: } define signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, i16 %a) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i16_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlv h0, v0.16b -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w8, w8, w0 -; CHECK-SD-BASE-NEXT: sxth w0, w8 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i16_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlv h0, v0.16b -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w8, w8, w0 -; CHECK-SD-DOT-NEXT: sxth w0, w8 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i16_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w8, w0, w8, uxth -; CHECK-GI-BASE-NEXT: sxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i16_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.8h, v1.8h, v0.16b -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w8, w0, w8, uxth -; CHECK-GI-DOT-NEXT: sxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i16_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlv h0, v0.16b +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w8, w8, w0 +; CHECK-SD-NEXT: sxth w0, w8 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i16_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: saddw2 v0.8h, v1.8h, v0.16b +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w8, w0, w8, uxth +; CHECK-GI-NEXT: sxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i16> %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) @@ -2232,41 +1567,23 @@ entry: } define zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, i16 %a) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i16_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w8, w8, w0 -; CHECK-SD-BASE-NEXT: and w0, w8, #0xffff -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i16_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w8, w8, w0 -; CHECK-SD-DOT-NEXT: and w0, w8, #0xffff -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i16_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w8, w0, w8, uxth -; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i16_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w8, w0, w8, uxth -; CHECK-GI-DOT-NEXT: and w0, w8, #0xffff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i16_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w8, w8, w0 +; CHECK-SD-NEXT: and w0, w8, #0xffff +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i16_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w8, w0, w8, uxth +; CHECK-GI-NEXT: and w0, w8, #0xffff +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i8> %x to <8 x i16> %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) @@ -2275,41 +1592,23 @@ entry: } define signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, i16 %a) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i16_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w8, w8, w0 -; CHECK-SD-BASE-NEXT: sxth w0, w8 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i16_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w8, w8, w0 -; CHECK-SD-DOT-NEXT: sxth w0, w8 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i16_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w8, w0, w8, uxth -; CHECK-GI-BASE-NEXT: sxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i16_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w8, w0, w8, uxth -; CHECK-GI-DOT-NEXT: sxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i16_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w8, w8, w0 +; CHECK-SD-NEXT: sxth w0, w8 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i16_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w8, w0, w8, uxth +; CHECK-GI-NEXT: sxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i8> %x to <8 x i16> %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) @@ -2318,37 +1617,21 @@ entry: } define zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, i8 %a) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i8_acc: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: addv b0, v0.16b -; CHECK-SD-BASE-NEXT: fmov w8, s0 -; CHECK-SD-BASE-NEXT: add w8, w8, w0 -; CHECK-SD-BASE-NEXT: and w0, w8, #0xff -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i8_acc: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: addv b0, v0.16b -; CHECK-SD-DOT-NEXT: fmov w8, s0 -; CHECK-SD-DOT-NEXT: add w8, w8, w0 -; CHECK-SD-DOT-NEXT: and w0, w8, #0xff -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i8_acc: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv b0, v0.16b -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: add w8, w0, w8, uxtb -; CHECK-GI-BASE-NEXT: and w0, w8, #0xff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i8_acc: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv b0, v0.16b -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: add w8, w0, w8, uxtb -; CHECK-GI-DOT-NEXT: and w0, w8, #0xff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i8_acc: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: addv b0, v0.16b +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w8, w8, w0 +; CHECK-SD-NEXT: and w0, w8, #0xff +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i8_acc: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv b0, v0.16b +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w8, w0, w8, uxtb +; CHECK-GI-NEXT: and w0, w8, #0xff +; CHECK-GI-NEXT: ret entry: %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x) %r = add i8 %z, %a @@ -2356,93 +1639,49 @@ entry: } define i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v2.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v3.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-BASE-NEXT: uaddl v2.2d, v3.2s, v2.2s -; CHECK-SD-BASE-NEXT: uaddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll2 v1.8h, v0.16b, #0 -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll2 v2.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v3.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-DOT-NEXT: uaddl v2.2d, v3.2s, v2.2s -; CHECK-SD-DOT-NEXT: uaddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v2.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v3.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v5.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v7.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-BASE-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-DOT-NEXT: ushll v2.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v3.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v5.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v7.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-DOT-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll2 v1.8h, v0.16b, #0 +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: ushll2 v2.4s, v1.8h, #0 +; CHECK-SD-NEXT: ushll2 v3.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddl2 v4.2d, v3.4s, v2.4s +; CHECK-SD-NEXT: uaddl v2.2d, v3.2s, v2.2s +; CHECK-SD-NEXT: uaddl2 v5.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ushll v2.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: ushll v5.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll v7.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v5.2d, v1.4s +; CHECK-GI-NEXT: uaddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v7.2d, v0.4s +; CHECK-GI-NEXT: add v1.2d, v2.2d, v1.2d +; CHECK-GI-NEXT: add v0.2d, v3.2d, v0.2d +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i64> %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) @@ -2451,93 +1690,49 @@ entry: } define i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v16i8_v16i64_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v2.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v3.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-BASE-NEXT: saddl v2.2d, v3.2s, v2.2s -; CHECK-SD-BASE-NEXT: saddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v16i8_v16i64_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll2 v1.8h, v0.16b, #0 -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: sshll2 v2.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v3.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v4.2d, v3.4s, v2.4s -; CHECK-SD-DOT-NEXT: saddl v2.2d, v3.2s, v2.2s -; CHECK-SD-DOT-NEXT: saddl2 v5.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v1.2d, v5.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v16i8_v16i64_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v2.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v3.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v5.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v7.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-BASE-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v16i8_v16i64_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v1.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-DOT-NEXT: sshll v2.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v3.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v5.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v7.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v5.2d, v1.4s -; CHECK-GI-DOT-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v7.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v1.2d, v2.2d, v1.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v16i8_v16i64_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll2 v1.8h, v0.16b, #0 +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll2 v2.4s, v1.8h, #0 +; CHECK-SD-NEXT: sshll2 v3.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddl2 v4.2d, v3.4s, v2.4s +; CHECK-SD-NEXT: saddl v2.2d, v3.2s, v2.2s +; CHECK-SD-NEXT: saddl2 v5.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v16i8_v16i64_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v1.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v2.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: sshll v5.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: sshll v7.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v5.2d, v1.4s +; CHECK-GI-NEXT: saddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v7.2d, v0.4s +; CHECK-GI-NEXT: add v1.2d, v2.2d, v1.2d +; CHECK-GI-NEXT: add v0.2d, v3.2d, v0.2d +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i64> %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) @@ -2546,61 +1741,33 @@ entry: } define i64 @add_v8i8_v8i64_acc_zext(<8 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: uaddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i8> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -2609,61 +1776,33 @@ entry: } define i64 @add_v8i8_v8i64_acc_sext(<8 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v8i8_v8i64_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v8i8_v8i64_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: sshll2 v1.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v2.2d, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v8i8_v8i64_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v8i8_v8i64_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll v1.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v2.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v2.2d, v1.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v3.2d, v0.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v1.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v8i8_v8i64_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v8i8_v8i64_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v2.2d, v1.2s, #0 +; CHECK-GI-NEXT: sshll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: saddw2 v1.2d, v2.2d, v1.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v3.2d, v0.4s +; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i8> %x to <8 x i64> %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -2672,51 +1811,28 @@ entry: } define i64 @add_v4i8_v4i64_acc_zext(<4 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v4i8_v4i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddlv d0, v0.4s -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i8_v4i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddlv d0, v0.4s -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i8_v4i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-BASE-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-GI-BASE-NEXT: and v2.16b, v2.16b, v1.16b -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i8_v4i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-DOT-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-GI-DOT-NEXT: and v2.16b, v2.16b, v1.16b -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i8_v4i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddlv d0, v0.4s +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i8_v4i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: movi v1.2d, #0x000000000000ff +; CHECK-GI-NEXT: ushll v2.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll2 v0.2d, v0.4s, #0 +; CHECK-GI-NEXT: and v2.16b, v2.16b, v1.16b +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i8> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -2725,61 +1841,33 @@ entry: } define i64 @add_v4i8_v4i64_acc_sext(<4 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v4i8_v4i64_acc_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-SD-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v4i8_v4i64_acc_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-SD-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v4i8_v4i64_acc_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.2d, v0.4s, #0 -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: addp d0, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v4i8_v4i64_acc_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.2d, v0.4s, #0 -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: ssra v1.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: addp d0, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v4i8_v4i64_acc_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll v1.2d, v0.2s, #0 +; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #0 +; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #56 +; CHECK-SD-NEXT: ssra v1.2d, v0.2d, #56 +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v4i8_v4i64_acc_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.2d, v0.4s, #0 +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: ssra v1.2d, v0.2d, #56 +; CHECK-GI-NEXT: addp d0, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i8> %x to <4 x i64> %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -2788,45 +1876,25 @@ entry: } define i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, i64 %a) { -; CHECK-SD-BASE-LABEL: add_v2i8_v2i64_acc_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: movi d1, #0x0000ff000000ff -; CHECK-SD-BASE-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x8, d0 -; CHECK-SD-BASE-NEXT: add x0, x8, x0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_v2i8_v2i64_acc_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: movi d1, #0x0000ff000000ff -; CHECK-SD-DOT-NEXT: and v0.8b, v0.8b, v1.8b -; CHECK-SD-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x8, d0 -; CHECK-SD-DOT-NEXT: add x0, x8, x0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_v2i8_v2i64_acc_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: add x0, x8, x0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_v2i8_v2i64_acc_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v1.2d, #0x000000000000ff -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: add x0, x8, x0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_v2i8_v2i64_acc_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi d1, #0x0000ff000000ff +; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x8, d0 +; CHECK-SD-NEXT: add x0, x8, x0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_v2i8_v2i64_acc_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v1.2d, #0x000000000000ff +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: add x0, x8, x0 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i8> %x to <2 x i64> %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -2865,37 +1933,21 @@ entry: } define i32 @add_pair_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i32_v4i32: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i32_v4i32: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i32_v4i32: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i32_v4i32: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i32_v4i32: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i32_v4i32: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x) %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y) @@ -2904,47 +1956,26 @@ entry: } define i64 @add_pair_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i32_v4i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlp v1.2d, v1.4s -; CHECK-SD-BASE-NEXT: uadalp v1.2d, v0.4s -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i32_v4i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlp v1.2d, v1.4s -; CHECK-SD-DOT-NEXT: uadalp v1.2d, v0.4s -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i32_v4i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v3.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i32_v4i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v3.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i32_v4i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlp v1.2d, v1.4s +; CHECK-SD-NEXT: uadalp v1.2d, v0.4s +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i32_v4i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v2.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v3.2d, v1.2s, #0 +; CHECK-GI-NEXT: uaddw2 v0.2d, v2.2d, v0.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i32> %x to <4 x i64> %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -2955,47 +1986,26 @@ entry: } define i64 @add_pair_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i32_v4i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlp v1.2d, v1.4s -; CHECK-SD-BASE-NEXT: sadalp v1.2d, v0.4s -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i32_v4i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlp v1.2d, v1.4s -; CHECK-SD-DOT-NEXT: sadalp v1.2d, v0.4s -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i32_v4i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v2.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v3.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i32_v4i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v2.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v3.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i32_v4i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlp v1.2d, v1.4s +; CHECK-SD-NEXT: sadalp v1.2d, v0.4s +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i32_v4i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v2.2d, v0.2s, #0 +; CHECK-GI-NEXT: sshll v3.2d, v1.2s, #0 +; CHECK-GI-NEXT: saddw2 v0.2d, v2.2d, v0.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i32> %x to <4 x i64> %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -3006,41 +2016,23 @@ entry: } define i64 @add_pair_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i32_v2i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v2i32_v2i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v2i32_v2i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v2i32_v2i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v2i32_v2i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i32_v2i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i32> %x to <2 x i64> %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -3051,41 +2043,23 @@ entry: } define i64 @add_pair_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i32_v2i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v2i32_v2i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v2i32_v2i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v2i32_v2i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v2i32_v2i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i32_v2i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: sshll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <2 x i32> %x to <2 x i64> %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -3096,47 +2070,26 @@ entry: } define i32 @add_pair_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i16_v8i32_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlp v1.4s, v1.8h -; CHECK-SD-BASE-NEXT: uadalp v1.4s, v0.8h -; CHECK-SD-BASE-NEXT: addv s0, v1.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i16_v8i32_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlp v1.4s, v1.8h -; CHECK-SD-DOT-NEXT: uadalp v1.4s, v0.8h -; CHECK-SD-DOT-NEXT: addv s0, v1.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i16_v8i32_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.4s, v2.4s, v0.8h -; CHECK-GI-BASE-NEXT: uaddw2 v1.4s, v3.4s, v1.8h -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i16_v8i32_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.4s, v2.4s, v0.8h -; CHECK-GI-DOT-NEXT: uaddw2 v1.4s, v3.4s, v1.8h -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i16_v8i32_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlp v1.4s, v1.8h +; CHECK-SD-NEXT: uadalp v1.4s, v0.8h +; CHECK-SD-NEXT: addv s0, v1.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i16_v8i32_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0 +; CHECK-GI-NEXT: uaddw2 v0.4s, v2.4s, v0.8h +; CHECK-GI-NEXT: uaddw2 v1.4s, v3.4s, v1.8h +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i16> %x to <8 x i32> %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) @@ -3147,47 +2100,26 @@ entry: } define i32 @add_pair_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i16_v8i32_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlp v1.4s, v1.8h -; CHECK-SD-BASE-NEXT: sadalp v1.4s, v0.8h -; CHECK-SD-BASE-NEXT: addv s0, v1.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i16_v8i32_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlp v1.4s, v1.8h -; CHECK-SD-DOT-NEXT: sadalp v1.4s, v0.8h -; CHECK-SD-DOT-NEXT: addv s0, v1.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i16_v8i32_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.4s, v2.4s, v0.8h -; CHECK-GI-BASE-NEXT: saddw2 v1.4s, v3.4s, v1.8h -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i16_v8i32_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.4s, v2.4s, v0.8h -; CHECK-GI-DOT-NEXT: saddw2 v1.4s, v3.4s, v1.8h -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i16_v8i32_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlp v1.4s, v1.8h +; CHECK-SD-NEXT: sadalp v1.4s, v0.8h +; CHECK-SD-NEXT: addv s0, v1.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i16_v8i32_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v2.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v1.4h, #0 +; CHECK-GI-NEXT: saddw2 v0.4s, v2.4s, v0.8h +; CHECK-GI-NEXT: saddw2 v1.4s, v3.4s, v1.8h +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i16> %x to <8 x i32> %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx) @@ -3198,41 +2130,23 @@ entry: } define i32 @add_pair_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i16_v4i32_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i16_v4i32_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddl v0.4s, v0.4h, v1.4h -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i16_v4i32_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i16_v4i32_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i16_v4i32_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddl v0.4s, v0.4h, v1.4h +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i16_v4i32_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i16> %x to <4 x i32> %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) @@ -3243,69 +2157,69 @@ entry: } define i32 @add_pair_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i16_v4i32_sext: +; CHECK-SD-LABEL: add_pair_v4i16_v4i32_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddl v0.4s, v0.4h, v1.4h +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i16_v4i32_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %xx = sext <4 x i16> %x to <4 x i32> + %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) + %yy = sext <4 x i16> %y to <4 x i32> + %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy) + %z = add i32 %z1, %z2 + ret i32 %z +} + +define i32 @test_udot_v8i8(<8 x i8> %a, <8 x i8> %b) { +; CHECK-SD-BASE-LABEL: test_udot_v8i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: addv s0, v0.4s +; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: umull v2.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: addv s0, v2.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-SD-DOT-LABEL: add_pair_v4i16_v4i32_sext: +; CHECK-SD-DOT-LABEL: test_udot_v8i8: ; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddl v0.4s, v0.4h, v1.4h -; CHECK-SD-DOT-NEXT: addv s0, v0.4s +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v0.8b +; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s ; CHECK-SD-DOT-NEXT: fmov w0, s0 ; CHECK-SD-DOT-NEXT: ret ; -; CHECK-GI-BASE-LABEL: add_pair_v4i16_v4i32_sext: +; CHECK-GI-BASE-LABEL: test_udot_v8i8: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 +; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-BASE-NEXT: umull v2.4s, v1.4h, v0.4h +; CHECK-GI-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h +; CHECK-GI-BASE-NEXT: addv s0, v2.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret ; -; CHECK-GI-DOT-LABEL: add_pair_v4i16_v4i32_sext: +; CHECK-GI-DOT-LABEL: test_udot_v8i8: ; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: udot v2.2s, v1.8b, v0.8b +; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s +; CHECK-GI-DOT-NEXT: fmov w0, s0 ; CHECK-GI-DOT-NEXT: ret entry: - %xx = sext <4 x i16> %x to <4 x i32> - %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) - %yy = sext <4 x i16> %y to <4 x i32> - %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy) - %z = add i32 %z1, %z2 - ret i32 %z -} - -define i32 @test_udot_v8i8(<8 x i8> %a, <8 x i8> %b) { -; CHECK-BASE-LABEL: test_udot_v8i8: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-BASE-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h -; CHECK-BASE-NEXT: addv s0, v2.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret -; -; CHECK-DOT-LABEL: test_udot_v8i8: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: udot v2.2s, v1.8b, v0.8b -; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret -entry: %0 = zext <8 x i8> %a to <8 x i32> %1 = zext <8 x i8> %b to <8 x i32> %2 = mul nuw nsw <8 x i32> %1, %0 @@ -3329,13 +2243,13 @@ define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-DOT-LABEL: test_udot_v16i8: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: udot v2.4s, v1.16b, v0.16b -; CHECK-DOT-NEXT: addv s0, v2.4s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-DOT-LABEL: test_udot_v16i8: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: udot v2.4s, v1.16b, v0.16b +; CHECK-SD-DOT-NEXT: addv s0, v2.4s +; CHECK-SD-DOT-NEXT: fmov w0, s0 +; CHECK-SD-DOT-NEXT: ret ; ; CHECK-GI-BASE-LABEL: test_udot_v16i8: ; CHECK-GI-BASE: // %bb.0: // %entry @@ -3351,6 +2265,14 @@ define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; CHECK-GI-BASE-NEXT: addv s0, v0.4s ; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: test_udot_v16i8: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: udot v2.4s, v1.16b, v0.16b +; CHECK-GI-DOT-NEXT: addv s0, v2.4s +; CHECK-GI-DOT-NEXT: fmov w0, s0 +; CHECK-GI-DOT-NEXT: ret entry: %0 = zext <16 x i8> %a to <16 x i32> %1 = zext <16 x i8> %b to <16 x i32> @@ -3360,28 +2282,28 @@ entry: } define i32 @test_udot_v24i8(ptr %p1, ptr %p2) { -; CHECK-BASE-LABEL: test_udot_v24i8: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ldr q0, [x0] -; CHECK-BASE-NEXT: ldr q1, [x1] -; CHECK-BASE-NEXT: ldr d4, [x0, #16] -; CHECK-BASE-NEXT: ldr d5, [x1, #16] -; CHECK-BASE-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-BASE-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-BASE-NEXT: umull v6.4s, v3.4h, v2.4h -; CHECK-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h -; CHECK-BASE-NEXT: ushll v3.8h, v4.8b, #0 -; CHECK-BASE-NEXT: ushll v4.8h, v5.8b, #0 -; CHECK-BASE-NEXT: umlal2 v2.4s, v4.8h, v3.8h -; CHECK-BASE-NEXT: umlal v6.4s, v4.4h, v3.4h -; CHECK-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h -; CHECK-BASE-NEXT: umlal v6.4s, v1.4h, v0.4h -; CHECK-BASE-NEXT: add v0.4s, v6.4s, v2.4s -; CHECK-BASE-NEXT: addv s0, v0.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret +; CHECK-SD-BASE-LABEL: test_udot_v24i8: +; CHECK-SD-BASE: // %bb.0: // %entry +; CHECK-SD-BASE-NEXT: ldr q0, [x0] +; CHECK-SD-BASE-NEXT: ldr q1, [x1] +; CHECK-SD-BASE-NEXT: ldr d4, [x0, #16] +; CHECK-SD-BASE-NEXT: ldr d5, [x1, #16] +; CHECK-SD-BASE-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-SD-BASE-NEXT: umull v6.4s, v3.4h, v2.4h +; CHECK-SD-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h +; CHECK-SD-BASE-NEXT: ushll v3.8h, v4.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v4.8h, v5.8b, #0 +; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v4.8h, v3.8h +; CHECK-SD-BASE-NEXT: umlal v6.4s, v4.4h, v3.4h +; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: umlal v6.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v6.4s, v2.4s +; CHECK-SD-BASE-NEXT: addv s0, v0.4s +; CHECK-SD-BASE-NEXT: fmov w0, s0 +; CHECK-SD-BASE-NEXT: ret ; ; CHECK-SD-DOT-LABEL: test_udot_v24i8: ; CHECK-SD-DOT: // %bb.0: // %entry @@ -3400,6 +2322,29 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) { ; CHECK-SD-DOT-NEXT: add w0, w9, w8 ; CHECK-SD-DOT-NEXT: ret ; +; CHECK-GI-BASE-LABEL: test_udot_v24i8: +; CHECK-GI-BASE: // %bb.0: // %entry +; CHECK-GI-BASE-NEXT: ldr q0, [x0] +; CHECK-GI-BASE-NEXT: ldr q1, [x1] +; CHECK-GI-BASE-NEXT: ldr d4, [x0, #16] +; CHECK-GI-BASE-NEXT: ldr d5, [x1, #16] +; CHECK-GI-BASE-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-GI-BASE-NEXT: umull v6.4s, v3.4h, v2.4h +; CHECK-GI-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h +; CHECK-GI-BASE-NEXT: ushll v3.8h, v4.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v4.8h, v5.8b, #0 +; CHECK-GI-BASE-NEXT: umlal2 v2.4s, v4.8h, v3.8h +; CHECK-GI-BASE-NEXT: umlal v6.4s, v4.4h, v3.4h +; CHECK-GI-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h +; CHECK-GI-BASE-NEXT: umlal v6.4s, v1.4h, v0.4h +; CHECK-GI-BASE-NEXT: add v0.4s, v6.4s, v2.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ret +; ; CHECK-GI-DOT-LABEL: test_udot_v24i8: ; CHECK-GI-DOT: // %bb.0: // %entry ; CHECK-GI-DOT-NEXT: ldr b1, [x0] @@ -3515,42 +2460,42 @@ entry: } define i32 @test_udot_v48i8(ptr %p1, ptr %p2) { -; CHECK-BASE-LABEL: test_udot_v48i8: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ldp q0, q4, [x1] -; CHECK-BASE-NEXT: ldr q2, [x0, #32] -; CHECK-BASE-NEXT: ldp q1, q3, [x0] -; CHECK-BASE-NEXT: ldr q7, [x1, #32] -; CHECK-BASE-NEXT: ushll2 v16.8h, v2.16b, #0 -; CHECK-BASE-NEXT: ushll2 v6.8h, v0.16b, #0 -; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-BASE-NEXT: ushll2 v17.8h, v7.16b, #0 -; CHECK-BASE-NEXT: ushll2 v5.8h, v1.16b, #0 -; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-BASE-NEXT: umull2 v18.4s, v6.8h, v5.8h -; CHECK-BASE-NEXT: umull v19.4s, v0.4h, v1.4h -; CHECK-BASE-NEXT: umull v5.4s, v6.4h, v5.4h -; CHECK-BASE-NEXT: umull2 v0.4s, v0.8h, v1.8h -; CHECK-BASE-NEXT: ushll v1.8h, v2.8b, #0 -; CHECK-BASE-NEXT: ushll v2.8h, v7.8b, #0 -; CHECK-BASE-NEXT: ushll2 v6.8h, v3.16b, #0 -; CHECK-BASE-NEXT: ushll2 v7.8h, v4.16b, #0 -; CHECK-BASE-NEXT: umlal2 v18.4s, v17.8h, v16.8h -; CHECK-BASE-NEXT: umlal v5.4s, v17.4h, v16.4h -; CHECK-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h -; CHECK-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h -; CHECK-BASE-NEXT: ushll v1.8h, v3.8b, #0 -; CHECK-BASE-NEXT: ushll v2.8h, v4.8b, #0 -; CHECK-BASE-NEXT: umlal2 v18.4s, v7.8h, v6.8h -; CHECK-BASE-NEXT: umlal v5.4s, v7.4h, v6.4h -; CHECK-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h -; CHECK-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h -; CHECK-BASE-NEXT: add v1.4s, v19.4s, v5.4s -; CHECK-BASE-NEXT: add v0.4s, v0.4s, v18.4s -; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-BASE-NEXT: addv s0, v0.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret +; CHECK-SD-BASE-LABEL: test_udot_v48i8: +; CHECK-SD-BASE: // %bb.0: // %entry +; CHECK-SD-BASE-NEXT: ldp q0, q4, [x1] +; CHECK-SD-BASE-NEXT: ldr q2, [x0, #32] +; CHECK-SD-BASE-NEXT: ldp q1, q3, [x0] +; CHECK-SD-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-SD-BASE-NEXT: ushll2 v16.8h, v2.16b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v6.8h, v0.16b, #0 +; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v17.8h, v7.16b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v5.8h, v1.16b, #0 +; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: umull2 v18.4s, v6.8h, v5.8h +; CHECK-SD-BASE-NEXT: umull v19.4s, v0.4h, v1.4h +; CHECK-SD-BASE-NEXT: umull v5.4s, v6.4h, v5.4h +; CHECK-SD-BASE-NEXT: umull2 v0.4s, v0.8h, v1.8h +; CHECK-SD-BASE-NEXT: ushll v1.8h, v2.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v2.8h, v7.8b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v6.8h, v3.16b, #0 +; CHECK-SD-BASE-NEXT: ushll2 v7.8h, v4.16b, #0 +; CHECK-SD-BASE-NEXT: umlal2 v18.4s, v17.8h, v16.8h +; CHECK-SD-BASE-NEXT: umlal v5.4s, v17.4h, v16.4h +; CHECK-SD-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h +; CHECK-SD-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h +; CHECK-SD-BASE-NEXT: ushll v1.8h, v3.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v2.8h, v4.8b, #0 +; CHECK-SD-BASE-NEXT: umlal2 v18.4s, v7.8h, v6.8h +; CHECK-SD-BASE-NEXT: umlal v5.4s, v7.4h, v6.4h +; CHECK-SD-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h +; CHECK-SD-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h +; CHECK-SD-BASE-NEXT: add v1.4s, v19.4s, v5.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v18.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-BASE-NEXT: addv s0, v0.4s +; CHECK-SD-BASE-NEXT: fmov w0, s0 +; CHECK-SD-BASE-NEXT: ret ; ; CHECK-SD-DOT-LABEL: test_udot_v48i8: ; CHECK-SD-DOT: // %bb.0: // %entry @@ -3566,6 +2511,43 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) { ; CHECK-SD-DOT-NEXT: fmov w0, s0 ; CHECK-SD-DOT-NEXT: ret ; +; CHECK-GI-BASE-LABEL: test_udot_v48i8: +; CHECK-GI-BASE: // %bb.0: // %entry +; CHECK-GI-BASE-NEXT: ldp q0, q4, [x1] +; CHECK-GI-BASE-NEXT: ldr q2, [x0, #32] +; CHECK-GI-BASE-NEXT: ldp q1, q3, [x0] +; CHECK-GI-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-GI-BASE-NEXT: ushll2 v16.8h, v2.16b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v0.16b, #0 +; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v17.8h, v7.16b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v5.8h, v1.16b, #0 +; CHECK-GI-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-BASE-NEXT: umull2 v18.4s, v6.8h, v5.8h +; CHECK-GI-BASE-NEXT: umull v19.4s, v0.4h, v1.4h +; CHECK-GI-BASE-NEXT: umull v5.4s, v6.4h, v5.4h +; CHECK-GI-BASE-NEXT: umull2 v0.4s, v0.8h, v1.8h +; CHECK-GI-BASE-NEXT: ushll v1.8h, v2.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v2.8h, v7.8b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v3.16b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v7.8h, v4.16b, #0 +; CHECK-GI-BASE-NEXT: umlal2 v18.4s, v17.8h, v16.8h +; CHECK-GI-BASE-NEXT: umlal v5.4s, v17.4h, v16.4h +; CHECK-GI-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h +; CHECK-GI-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h +; CHECK-GI-BASE-NEXT: ushll v1.8h, v3.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v2.8h, v4.8b, #0 +; CHECK-GI-BASE-NEXT: umlal2 v18.4s, v7.8h, v6.8h +; CHECK-GI-BASE-NEXT: umlal v5.4s, v7.4h, v6.4h +; CHECK-GI-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h +; CHECK-GI-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h +; CHECK-GI-BASE-NEXT: add v1.4s, v19.4s, v5.4s +; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v18.4s +; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ret +; ; CHECK-GI-DOT-LABEL: test_udot_v48i8: ; CHECK-GI-DOT: // %bb.0: // %entry ; CHECK-GI-DOT-NEXT: ldr b1, [x0] @@ -3780,23 +2762,41 @@ entry: } define i32 @test_sdot_v8i8(<8 x i8> %a, <8 x i8> %b) { -; CHECK-BASE-LABEL: test_sdot_v8i8: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-BASE-NEXT: smull v2.4s, v1.4h, v0.4h -; CHECK-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h -; CHECK-BASE-NEXT: addv s0, v2.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret -; -; CHECK-DOT-LABEL: test_sdot_v8i8: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: sdot v2.2s, v1.8b, v0.8b -; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-BASE-LABEL: test_sdot_v8i8: +; CHECK-SD-BASE: // %bb.0: // %entry +; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: smull v2.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: addv s0, v2.4s +; CHECK-SD-BASE-NEXT: fmov w0, s0 +; CHECK-SD-BASE-NEXT: ret +; +; CHECK-SD-DOT-LABEL: test_sdot_v8i8: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: sdot v2.2s, v1.8b, v0.8b +; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s +; CHECK-SD-DOT-NEXT: fmov w0, s0 +; CHECK-SD-DOT-NEXT: ret +; +; CHECK-GI-BASE-LABEL: test_sdot_v8i8: +; CHECK-GI-BASE: // %bb.0: // %entry +; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-BASE-NEXT: smull v2.4s, v1.4h, v0.4h +; CHECK-GI-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h +; CHECK-GI-BASE-NEXT: addv s0, v2.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: test_sdot_v8i8: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: sdot v2.2s, v1.8b, v0.8b +; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s +; CHECK-GI-DOT-NEXT: fmov w0, s0 +; CHECK-GI-DOT-NEXT: ret entry: %0 = sext <8 x i8> %a to <8 x i32> %1 = sext <8 x i8> %b to <8 x i32> @@ -3821,13 +2821,13 @@ define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-DOT-LABEL: test_sdot_v16i8: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: sdot v2.4s, v1.16b, v0.16b -; CHECK-DOT-NEXT: addv s0, v2.4s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret +; CHECK-SD-DOT-LABEL: test_sdot_v16i8: +; CHECK-SD-DOT: // %bb.0: // %entry +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: sdot v2.4s, v1.16b, v0.16b +; CHECK-SD-DOT-NEXT: addv s0, v2.4s +; CHECK-SD-DOT-NEXT: fmov w0, s0 +; CHECK-SD-DOT-NEXT: ret ; ; CHECK-GI-BASE-LABEL: test_sdot_v16i8: ; CHECK-GI-BASE: // %bb.0: // %entry @@ -3843,6 +2843,14 @@ define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; CHECK-GI-BASE-NEXT: addv s0, v0.4s ; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret +; +; CHECK-GI-DOT-LABEL: test_sdot_v16i8: +; CHECK-GI-DOT: // %bb.0: // %entry +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: sdot v2.4s, v1.16b, v0.16b +; CHECK-GI-DOT-NEXT: addv s0, v2.4s +; CHECK-GI-DOT-NEXT: fmov w0, s0 +; CHECK-GI-DOT-NEXT: ret entry: %0 = sext <16 x i8> %a to <16 x i32> %1 = sext <16 x i8> %b to <16 x i32> @@ -3852,28 +2860,28 @@ entry: } define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) { -; CHECK-BASE-LABEL: test_sdot_v24i8: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ldr q0, [x0] -; CHECK-BASE-NEXT: ldr q1, [x1] -; CHECK-BASE-NEXT: ldr d4, [x0, #16] -; CHECK-BASE-NEXT: ldr d5, [x1, #16] -; CHECK-BASE-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-BASE-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-BASE-NEXT: smull v6.4s, v3.4h, v2.4h -; CHECK-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h -; CHECK-BASE-NEXT: sshll v3.8h, v4.8b, #0 -; CHECK-BASE-NEXT: sshll v4.8h, v5.8b, #0 -; CHECK-BASE-NEXT: smlal2 v2.4s, v4.8h, v3.8h -; CHECK-BASE-NEXT: smlal v6.4s, v4.4h, v3.4h -; CHECK-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h -; CHECK-BASE-NEXT: smlal v6.4s, v1.4h, v0.4h -; CHECK-BASE-NEXT: add v0.4s, v6.4s, v2.4s -; CHECK-BASE-NEXT: addv s0, v0.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret +; CHECK-SD-BASE-LABEL: test_sdot_v24i8: +; CHECK-SD-BASE: // %bb.0: // %entry +; CHECK-SD-BASE-NEXT: ldr q0, [x0] +; CHECK-SD-BASE-NEXT: ldr q1, [x1] +; CHECK-SD-BASE-NEXT: ldr d4, [x0, #16] +; CHECK-SD-BASE-NEXT: ldr d5, [x1, #16] +; CHECK-SD-BASE-NEXT: sshll v2.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v3.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-SD-BASE-NEXT: smull v6.4s, v3.4h, v2.4h +; CHECK-SD-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h +; CHECK-SD-BASE-NEXT: sshll v3.8h, v4.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v4.8h, v5.8b, #0 +; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v4.8h, v3.8h +; CHECK-SD-BASE-NEXT: smlal v6.4s, v4.4h, v3.4h +; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v6.4s, v2.4s +; CHECK-SD-BASE-NEXT: addv s0, v0.4s +; CHECK-SD-BASE-NEXT: fmov w0, s0 +; CHECK-SD-BASE-NEXT: ret ; ; CHECK-SD-DOT-LABEL: test_sdot_v24i8: ; CHECK-SD-DOT: // %bb.0: // %entry @@ -3892,6 +2900,29 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) { ; CHECK-SD-DOT-NEXT: add w0, w9, w8 ; CHECK-SD-DOT-NEXT: ret ; +; CHECK-GI-BASE-LABEL: test_sdot_v24i8: +; CHECK-GI-BASE: // %bb.0: // %entry +; CHECK-GI-BASE-NEXT: ldr q0, [x0] +; CHECK-GI-BASE-NEXT: ldr q1, [x1] +; CHECK-GI-BASE-NEXT: ldr d4, [x0, #16] +; CHECK-GI-BASE-NEXT: ldr d5, [x1, #16] +; CHECK-GI-BASE-NEXT: sshll v2.8h, v0.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v3.8h, v1.8b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-GI-BASE-NEXT: smull v6.4s, v3.4h, v2.4h +; CHECK-GI-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h +; CHECK-GI-BASE-NEXT: sshll v3.8h, v4.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v4.8h, v5.8b, #0 +; CHECK-GI-BASE-NEXT: smlal2 v2.4s, v4.8h, v3.8h +; CHECK-GI-BASE-NEXT: smlal v6.4s, v4.4h, v3.4h +; CHECK-GI-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h +; CHECK-GI-BASE-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-GI-BASE-NEXT: add v0.4s, v6.4s, v2.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ret +; ; CHECK-GI-DOT-LABEL: test_sdot_v24i8: ; CHECK-GI-DOT: // %bb.0: // %entry ; CHECK-GI-DOT-NEXT: ldr b1, [x0] @@ -4007,42 +3038,42 @@ entry: } define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) { -; CHECK-BASE-LABEL: test_sdot_v48i8: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ldp q0, q4, [x1] -; CHECK-BASE-NEXT: ldr q2, [x0, #32] -; CHECK-BASE-NEXT: ldp q1, q3, [x0] -; CHECK-BASE-NEXT: ldr q7, [x1, #32] -; CHECK-BASE-NEXT: sshll2 v16.8h, v2.16b, #0 -; CHECK-BASE-NEXT: sshll2 v6.8h, v0.16b, #0 -; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-BASE-NEXT: sshll2 v17.8h, v7.16b, #0 -; CHECK-BASE-NEXT: sshll2 v5.8h, v1.16b, #0 -; CHECK-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-BASE-NEXT: smull2 v18.4s, v6.8h, v5.8h -; CHECK-BASE-NEXT: smull v19.4s, v0.4h, v1.4h -; CHECK-BASE-NEXT: smull v5.4s, v6.4h, v5.4h -; CHECK-BASE-NEXT: smull2 v0.4s, v0.8h, v1.8h -; CHECK-BASE-NEXT: sshll v1.8h, v2.8b, #0 -; CHECK-BASE-NEXT: sshll v2.8h, v7.8b, #0 -; CHECK-BASE-NEXT: sshll2 v6.8h, v3.16b, #0 -; CHECK-BASE-NEXT: sshll2 v7.8h, v4.16b, #0 -; CHECK-BASE-NEXT: smlal2 v18.4s, v17.8h, v16.8h -; CHECK-BASE-NEXT: smlal v5.4s, v17.4h, v16.4h -; CHECK-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h -; CHECK-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h -; CHECK-BASE-NEXT: sshll v1.8h, v3.8b, #0 -; CHECK-BASE-NEXT: sshll v2.8h, v4.8b, #0 -; CHECK-BASE-NEXT: smlal2 v18.4s, v7.8h, v6.8h -; CHECK-BASE-NEXT: smlal v5.4s, v7.4h, v6.4h -; CHECK-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h -; CHECK-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h -; CHECK-BASE-NEXT: add v1.4s, v19.4s, v5.4s -; CHECK-BASE-NEXT: add v0.4s, v0.4s, v18.4s -; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-BASE-NEXT: addv s0, v0.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret +; CHECK-SD-BASE-LABEL: test_sdot_v48i8: +; CHECK-SD-BASE: // %bb.0: // %entry +; CHECK-SD-BASE-NEXT: ldp q0, q4, [x1] +; CHECK-SD-BASE-NEXT: ldr q2, [x0, #32] +; CHECK-SD-BASE-NEXT: ldp q1, q3, [x0] +; CHECK-SD-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-SD-BASE-NEXT: sshll2 v16.8h, v2.16b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v6.8h, v0.16b, #0 +; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v17.8h, v7.16b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v5.8h, v1.16b, #0 +; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: smull2 v18.4s, v6.8h, v5.8h +; CHECK-SD-BASE-NEXT: smull v19.4s, v0.4h, v1.4h +; CHECK-SD-BASE-NEXT: smull v5.4s, v6.4h, v5.4h +; CHECK-SD-BASE-NEXT: smull2 v0.4s, v0.8h, v1.8h +; CHECK-SD-BASE-NEXT: sshll v1.8h, v2.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v2.8h, v7.8b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v6.8h, v3.16b, #0 +; CHECK-SD-BASE-NEXT: sshll2 v7.8h, v4.16b, #0 +; CHECK-SD-BASE-NEXT: smlal2 v18.4s, v17.8h, v16.8h +; CHECK-SD-BASE-NEXT: smlal v5.4s, v17.4h, v16.4h +; CHECK-SD-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h +; CHECK-SD-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h +; CHECK-SD-BASE-NEXT: sshll v1.8h, v3.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v2.8h, v4.8b, #0 +; CHECK-SD-BASE-NEXT: smlal2 v18.4s, v7.8h, v6.8h +; CHECK-SD-BASE-NEXT: smlal v5.4s, v7.4h, v6.4h +; CHECK-SD-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h +; CHECK-SD-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h +; CHECK-SD-BASE-NEXT: add v1.4s, v19.4s, v5.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v18.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-BASE-NEXT: addv s0, v0.4s +; CHECK-SD-BASE-NEXT: fmov w0, s0 +; CHECK-SD-BASE-NEXT: ret ; ; CHECK-SD-DOT-LABEL: test_sdot_v48i8: ; CHECK-SD-DOT: // %bb.0: // %entry @@ -4058,6 +3089,43 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) { ; CHECK-SD-DOT-NEXT: fmov w0, s0 ; CHECK-SD-DOT-NEXT: ret ; +; CHECK-GI-BASE-LABEL: test_sdot_v48i8: +; CHECK-GI-BASE: // %bb.0: // %entry +; CHECK-GI-BASE-NEXT: ldp q0, q4, [x1] +; CHECK-GI-BASE-NEXT: ldr q2, [x0, #32] +; CHECK-GI-BASE-NEXT: ldp q1, q3, [x0] +; CHECK-GI-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-GI-BASE-NEXT: sshll2 v16.8h, v2.16b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v0.16b, #0 +; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v17.8h, v7.16b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v5.8h, v1.16b, #0 +; CHECK-GI-BASE-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-BASE-NEXT: smull2 v18.4s, v6.8h, v5.8h +; CHECK-GI-BASE-NEXT: smull v19.4s, v0.4h, v1.4h +; CHECK-GI-BASE-NEXT: smull v5.4s, v6.4h, v5.4h +; CHECK-GI-BASE-NEXT: smull2 v0.4s, v0.8h, v1.8h +; CHECK-GI-BASE-NEXT: sshll v1.8h, v2.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v2.8h, v7.8b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v3.16b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v7.8h, v4.16b, #0 +; CHECK-GI-BASE-NEXT: smlal2 v18.4s, v17.8h, v16.8h +; CHECK-GI-BASE-NEXT: smlal v5.4s, v17.4h, v16.4h +; CHECK-GI-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h +; CHECK-GI-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h +; CHECK-GI-BASE-NEXT: sshll v1.8h, v3.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v2.8h, v4.8b, #0 +; CHECK-GI-BASE-NEXT: smlal2 v18.4s, v7.8h, v6.8h +; CHECK-GI-BASE-NEXT: smlal v5.4s, v7.4h, v6.4h +; CHECK-GI-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h +; CHECK-GI-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h +; CHECK-GI-BASE-NEXT: add v1.4s, v19.4s, v5.4s +; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v18.4s +; CHECK-GI-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 +; CHECK-GI-BASE-NEXT: ret +; ; CHECK-GI-DOT-LABEL: test_sdot_v48i8: ; CHECK-GI-DOT: // %bb.0: // %entry ; CHECK-GI-DOT-NEXT: ldr b1, [x0] @@ -4273,18 +3341,18 @@ entry: ; Test to ensure that if G_MUL has more than 1 use, it should not be combined to UDOT define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) { -; CHECK-BASE-LABEL: test_udot_v8i8_multi_use: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-BASE-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-BASE-NEXT: mov v3.16b, v2.16b -; CHECK-BASE-NEXT: fmov w8, s2 -; CHECK-BASE-NEXT: umlal2 v3.4s, v1.8h, v0.8h -; CHECK-BASE-NEXT: addv s0, v3.4s -; CHECK-BASE-NEXT: fmov w9, s0 -; CHECK-BASE-NEXT: add w0, w9, w8 -; CHECK-BASE-NEXT: ret +; CHECK-SD-BASE-LABEL: test_udot_v8i8_multi_use: +; CHECK-SD-BASE: // %bb.0: // %entry +; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: umull v2.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: mov v3.16b, v2.16b +; CHECK-SD-BASE-NEXT: fmov w8, s2 +; CHECK-SD-BASE-NEXT: umlal2 v3.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: addv s0, v3.4s +; CHECK-SD-BASE-NEXT: fmov w9, s0 +; CHECK-SD-BASE-NEXT: add w0, w9, w8 +; CHECK-SD-BASE-NEXT: ret ; ; CHECK-SD-DOT-LABEL: test_udot_v8i8_multi_use: ; CHECK-SD-DOT: // %bb.0: // %entry @@ -4299,18 +3367,18 @@ define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-DOT-NEXT: add w0, w8, w9 ; CHECK-SD-DOT-NEXT: ret ; -; CHECK-GI-DOT-LABEL: test_udot_v8i8_multi_use: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-GI-DOT-NEXT: mov v3.16b, v2.16b -; CHECK-GI-DOT-NEXT: fmov w8, s2 -; CHECK-GI-DOT-NEXT: umlal2 v3.4s, v1.8h, v0.8h -; CHECK-GI-DOT-NEXT: addv s0, v3.4s -; CHECK-GI-DOT-NEXT: fmov w9, s0 -; CHECK-GI-DOT-NEXT: add w0, w9, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-GI-LABEL: test_udot_v8i8_multi_use: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: umull v2.4s, v1.4h, v0.4h +; CHECK-GI-NEXT: mov v3.16b, v2.16b +; CHECK-GI-NEXT: fmov w8, s2 +; CHECK-GI-NEXT: umlal2 v3.4s, v1.8h, v0.8h +; CHECK-GI-NEXT: addv s0, v3.4s +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: add w0, w9, w8 +; CHECK-GI-NEXT: ret entry: %0 = zext <8 x i8> %a to <8 x i32> %1 = zext <8 x i8> %b to <8 x i32> @@ -4322,39 +3390,22 @@ entry: } define zeroext i16 @add_pair_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i16_v8i16: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: add v0.8h, v0.8h, v1.8h -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i16_v8i16: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: add v0.8h, v0.8h, v1.8h -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i16_v8i16: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: addv h1, v1.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w8, w9, w8, uxth -; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i16_v8i16: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: addv h1, v1.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w8, w9, w8, uxth -; CHECK-GI-DOT-NEXT: and w0, w8, #0xffff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i16_v8i16: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: add v0.8h, v0.8h, v1.8h +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i16_v8i16: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: addv h1, v1.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w9, w8, uxth +; CHECK-GI-NEXT: and w0, w8, #0xffff +; CHECK-GI-NEXT: ret entry: %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x) %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %y) @@ -4363,85 +3414,45 @@ entry: } define i64 @add_pair_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i16_v8i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll2 v2.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v3.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v2.2s -; CHECK-SD-BASE-NEXT: uaddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-BASE-NEXT: uaddl v1.2d, v1.2s, v3.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i16_v8i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll2 v2.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v3.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v2.2s -; CHECK-SD-DOT-NEXT: uaddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-DOT-NEXT: uaddl v1.2d, v1.2s, v3.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i16_v8i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v5.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v7.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-BASE-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i16_v8i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v5.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v7.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-DOT-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i16_v8i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll2 v3.4s, v1.8h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: uaddl2 v4.2d, v0.4s, v2.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v2.2s +; CHECK-SD-NEXT: uaddl2 v2.2d, v1.4s, v3.4s +; CHECK-SD-NEXT: uaddl v1.2d, v1.2s, v3.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i16_v8i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: ushll v5.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll v7.2d, v1.2s, #0 +; CHECK-GI-NEXT: uaddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v5.2d, v0.4s +; CHECK-GI-NEXT: uaddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v7.2d, v1.4s +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v3.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i16> %x to <8 x i64> %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -4452,85 +3463,45 @@ entry: } define i64 @add_pair_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i16_v8i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll2 v2.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v3.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v2.2s -; CHECK-SD-BASE-NEXT: saddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-BASE-NEXT: saddl v1.2d, v1.2s, v3.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i16_v8i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll2 v2.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v3.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v2.2s -; CHECK-SD-DOT-NEXT: saddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-DOT-NEXT: saddl v1.2d, v1.2s, v3.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i16_v8i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v5.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v7.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-BASE-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i16_v8i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v5.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v7.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-DOT-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i16_v8i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: sshll2 v3.4s, v1.8h, #0 +; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: saddl2 v4.2d, v0.4s, v2.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v2.2s +; CHECK-SD-NEXT: saddl2 v2.2d, v1.4s, v3.4s +; CHECK-SD-NEXT: saddl v1.2d, v1.2s, v3.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i16_v8i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v2.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: sshll v5.2d, v0.2s, #0 +; CHECK-GI-NEXT: sshll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: sshll v7.2d, v1.2s, #0 +; CHECK-GI-NEXT: saddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v5.2d, v0.4s +; CHECK-GI-NEXT: saddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v7.2d, v1.4s +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v3.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i16> %x to <8 x i64> %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -4541,55 +3512,30 @@ entry: } define i64 @add_pair_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i16_v4i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddlp v1.2d, v1.4s -; CHECK-SD-BASE-NEXT: uadalp v1.2d, v0.4s -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i16_v4i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddlp v1.2d, v1.4s -; CHECK-SD-DOT-NEXT: uadalp v1.2d, v0.4s -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i16_v4i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v3.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i16_v4i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v3.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i16_v4i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddlp v1.2d, v1.4s +; CHECK-SD-NEXT: uadalp v1.2d, v0.4s +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i16_v4i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll v2.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v3.2d, v1.2s, #0 +; CHECK-GI-NEXT: uaddw2 v0.2d, v2.2d, v0.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i16> %x to <4 x i64> %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -4600,55 +3546,30 @@ entry: } define i64 @add_pair_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i16_v4i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: saddlp v1.2d, v1.4s -; CHECK-SD-BASE-NEXT: sadalp v1.2d, v0.4s -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i16_v4i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: saddlp v1.2d, v1.4s -; CHECK-SD-DOT-NEXT: sadalp v1.2d, v0.4s -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i16_v4i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: sshll v2.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v3.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i16_v4i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: sshll v2.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v3.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v2.2d, v0.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v3.2d, v1.4s -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i16_v4i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddlp v1.2d, v1.4s +; CHECK-SD-NEXT: sadalp v1.2d, v0.4s +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i16_v4i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll v2.2d, v0.2s, #0 +; CHECK-GI-NEXT: sshll v3.2d, v1.2s, #0 +; CHECK-GI-NEXT: saddw2 v0.2d, v2.2d, v0.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v3.2d, v1.4s +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i16> %x to <4 x i64> %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -4659,53 +3580,29 @@ entry: } define i64 @add_pair_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i16_v2i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: movi d2, #0x00ffff0000ffff -; CHECK-SD-BASE-NEXT: and v0.8b, v0.8b, v2.8b -; CHECK-SD-BASE-NEXT: and v1.8b, v1.8b, v2.8b -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v2i16_v2i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: movi d2, #0x00ffff0000ffff -; CHECK-SD-DOT-NEXT: and v0.8b, v0.8b, v2.8b -; CHECK-SD-DOT-NEXT: and v1.8b, v1.8b, v2.8b -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v2i16_v2i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v2.2d, #0x0000000000ffff -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-BASE-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v2i16_v2i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v2.2d, #0x0000000000ffff -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-DOT-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v2i16_v2i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi d2, #0x00ffff0000ffff +; CHECK-SD-NEXT: and v0.8b, v0.8b, v2.8b +; CHECK-SD-NEXT: and v1.8b, v1.8b, v2.8b +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i16_v2i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v2.2d, #0x0000000000ffff +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i16> %x to <2 x i64> %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -4716,59 +3613,32 @@ entry: } define i64 @add_pair_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i16_v2i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-SD-BASE-NEXT: shl v0.2d, v0.2d, #48 -; CHECK-SD-BASE-NEXT: shl v1.2d, v1.2d, #48 -; CHECK-SD-BASE-NEXT: sshr v0.2d, v0.2d, #48 -; CHECK-SD-BASE-NEXT: ssra v0.2d, v1.2d, #48 -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v2i16_v2i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-SD-DOT-NEXT: shl v0.2d, v0.2d, #48 -; CHECK-SD-DOT-NEXT: shl v1.2d, v1.2d, #48 -; CHECK-SD-DOT-NEXT: sshr v0.2d, v0.2d, #48 -; CHECK-SD-DOT-NEXT: ssra v0.2d, v1.2d, #48 -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v2i16_v2i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: shl v0.2d, v0.2d, #48 -; CHECK-GI-BASE-NEXT: shl v1.2d, v1.2d, #48 -; CHECK-GI-BASE-NEXT: sshr v0.2d, v0.2d, #48 -; CHECK-GI-BASE-NEXT: sshr v1.2d, v1.2d, #48 -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v2i16_v2i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: shl v0.2d, v0.2d, #48 -; CHECK-GI-DOT-NEXT: shl v1.2d, v1.2d, #48 -; CHECK-GI-DOT-NEXT: sshr v0.2d, v0.2d, #48 -; CHECK-GI-DOT-NEXT: sshr v1.2d, v1.2d, #48 -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v2i16_v2i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-SD-NEXT: shl v0.2d, v0.2d, #48 +; CHECK-SD-NEXT: shl v1.2d, v1.2d, #48 +; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #48 +; CHECK-SD-NEXT: ssra v0.2d, v1.2d, #48 +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i16_v2i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: shl v0.2d, v0.2d, #48 +; CHECK-GI-NEXT: shl v1.2d, v1.2d, #48 +; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #48 +; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #48 +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <2 x i16> %x to <2 x i64> %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -5041,51 +3911,28 @@ entry: } define i32 @add_pair_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i8_v4i32_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: bic v1.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i8_v4i32_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: bic v1.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: uaddl v0.4s, v0.4h, v1.4h -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i8_v4i32_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v2.2d, #0x0000ff000000ff -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-BASE-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i8_v4i32_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v2.2d, #0x0000ff000000ff -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-DOT-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i8_v4i32_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: bic v1.4h, #255, lsl #8 +; CHECK-SD-NEXT: uaddl v0.4s, v0.4h, v1.4h +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i8_v4i32_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v2.2d, #0x0000ff000000ff +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i8> %x to <4 x i32> %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) @@ -5096,59 +3943,32 @@ entry: } define i32 @add_pair_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i8_v4i32_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: shl v0.4s, v0.4s, #24 -; CHECK-SD-BASE-NEXT: shl v1.4s, v1.4s, #24 -; CHECK-SD-BASE-NEXT: sshr v0.4s, v0.4s, #24 -; CHECK-SD-BASE-NEXT: ssra v0.4s, v1.4s, #24 -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i8_v4i32_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: shl v0.4s, v0.4s, #24 -; CHECK-SD-DOT-NEXT: shl v1.4s, v1.4s, #24 -; CHECK-SD-DOT-NEXT: sshr v0.4s, v0.4s, #24 -; CHECK-SD-DOT-NEXT: ssra v0.4s, v1.4s, #24 -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i8_v4i32_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: shl v0.4s, v0.4s, #24 -; CHECK-GI-BASE-NEXT: shl v1.4s, v1.4s, #24 -; CHECK-GI-BASE-NEXT: sshr v0.4s, v0.4s, #24 -; CHECK-GI-BASE-NEXT: sshr v1.4s, v1.4s, #24 -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s1, v1.4s -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w0, w8, w9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i8_v4i32_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: shl v0.4s, v0.4s, #24 -; CHECK-GI-DOT-NEXT: shl v1.4s, v1.4s, #24 -; CHECK-GI-DOT-NEXT: sshr v0.4s, v0.4s, #24 -; CHECK-GI-DOT-NEXT: sshr v1.4s, v1.4s, #24 -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: addv s1, v1.4s -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w0, w8, w9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i8_v4i32_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24 +; CHECK-SD-NEXT: shl v1.4s, v1.4s, #24 +; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24 +; CHECK-SD-NEXT: ssra v0.4s, v1.4s, #24 +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i8_v4i32_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: shl v0.4s, v0.4s, #24 +; CHECK-GI-NEXT: shl v1.4s, v1.4s, #24 +; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #24 +; CHECK-GI-NEXT: sshr v1.4s, v1.4s, #24 +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i8> %x to <4 x i32> %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx) @@ -5159,49 +3979,27 @@ entry: } define zeroext i16 @add_pair_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i16_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlp v1.8h, v1.16b -; CHECK-SD-BASE-NEXT: uadalp v1.8h, v0.16b -; CHECK-SD-BASE-NEXT: addv h0, v1.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i16_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlp v1.8h, v1.16b -; CHECK-SD-DOT-NEXT: uadalp v1.8h, v0.16b -; CHECK-SD-DOT-NEXT: addv h0, v1.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i16_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v0.8h, v2.8h, v0.16b -; CHECK-GI-BASE-NEXT: uaddw2 v1.8h, v3.8h, v1.16b -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: addv h1, v1.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w8, w9, w8, uxth -; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i16_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v0.8h, v2.8h, v0.16b -; CHECK-GI-DOT-NEXT: uaddw2 v1.8h, v3.8h, v1.16b -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: addv h1, v1.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w8, w9, w8, uxth -; CHECK-GI-DOT-NEXT: and w0, w8, #0xffff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v16i8_v16i16_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlp v1.8h, v1.16b +; CHECK-SD-NEXT: uadalp v1.8h, v0.16b +; CHECK-SD-NEXT: addv h0, v1.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v16i8_v16i16_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-GI-NEXT: uaddw2 v0.8h, v2.8h, v0.16b +; CHECK-GI-NEXT: uaddw2 v1.8h, v3.8h, v1.16b +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: addv h1, v1.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w9, w8, uxth +; CHECK-GI-NEXT: and w0, w8, #0xffff +; CHECK-GI-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i16> %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) @@ -5212,49 +4010,27 @@ entry: } define signext i16 @add_pair_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i16_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddlp v1.8h, v1.16b -; CHECK-SD-BASE-NEXT: sadalp v1.8h, v0.16b -; CHECK-SD-BASE-NEXT: addv h0, v1.8h -; CHECK-SD-BASE-NEXT: smov w0, v0.h[0] -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i16_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddlp v1.8h, v1.16b -; CHECK-SD-DOT-NEXT: sadalp v1.8h, v0.16b -; CHECK-SD-DOT-NEXT: addv h0, v1.8h -; CHECK-SD-DOT-NEXT: smov w0, v0.h[0] -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i16_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: saddw2 v0.8h, v2.8h, v0.16b -; CHECK-GI-BASE-NEXT: saddw2 v1.8h, v3.8h, v1.16b -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: addv h1, v1.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w8, w9, w8, uxth -; CHECK-GI-BASE-NEXT: sxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i16_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: saddw2 v0.8h, v2.8h, v0.16b -; CHECK-GI-DOT-NEXT: saddw2 v1.8h, v3.8h, v1.16b -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: addv h1, v1.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w8, w9, w8, uxth -; CHECK-GI-DOT-NEXT: sxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v16i8_v16i16_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddlp v1.8h, v1.16b +; CHECK-SD-NEXT: sadalp v1.8h, v0.16b +; CHECK-SD-NEXT: addv h0, v1.8h +; CHECK-SD-NEXT: smov w0, v0.h[0] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v16i8_v16i16_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v3.8h, v1.8b, #0 +; CHECK-GI-NEXT: saddw2 v0.8h, v2.8h, v0.16b +; CHECK-GI-NEXT: saddw2 v1.8h, v3.8h, v1.16b +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: addv h1, v1.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w9, w8, uxth +; CHECK-GI-NEXT: sxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i16> %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx) @@ -5265,43 +4041,24 @@ entry: } define zeroext i16 @add_pair_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i16_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddl v0.8h, v0.8b, v1.8b -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i16_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddl v0.8h, v0.8b, v1.8b -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i16_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: addv h1, v1.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w8, w9, w8, uxth -; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i16_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: addv h1, v1.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w8, w9, w8, uxth -; CHECK-GI-DOT-NEXT: and w0, w8, #0xffff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i8_v8i16_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i8_v8i16_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: addv h1, v1.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w9, w8, uxth +; CHECK-GI-NEXT: and w0, w8, #0xffff +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i8> %x to <8 x i16> %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) @@ -5312,43 +4069,24 @@ entry: } define signext i16 @add_pair_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i16_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: saddl v0.8h, v0.8b, v1.8b -; CHECK-SD-BASE-NEXT: addv h0, v0.8h -; CHECK-SD-BASE-NEXT: smov w0, v0.h[0] -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i16_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: saddl v0.8h, v0.8b, v1.8b -; CHECK-SD-DOT-NEXT: addv h0, v0.8h -; CHECK-SD-DOT-NEXT: smov w0, v0.h[0] -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i16_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: addv h0, v0.8h -; CHECK-GI-BASE-NEXT: addv h1, v1.8h -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w8, w9, w8, uxth -; CHECK-GI-BASE-NEXT: sxth w0, w8 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i16_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: addv h0, v0.8h -; CHECK-GI-DOT-NEXT: addv h1, v1.8h -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w8, w9, w8, uxth -; CHECK-GI-DOT-NEXT: sxth w0, w8 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i8_v8i16_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: saddl v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: addv h0, v0.8h +; CHECK-SD-NEXT: smov w0, v0.h[0] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i8_v8i16_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: addv h0, v0.8h +; CHECK-GI-NEXT: addv h1, v1.8h +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w9, w8, uxth +; CHECK-GI-NEXT: sxth w0, w8 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i8> %x to <8 x i16> %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx) @@ -5359,39 +4097,22 @@ entry: } define zeroext i8 @add_pair_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i8: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: add v0.16b, v0.16b, v1.16b -; CHECK-SD-BASE-NEXT: addv b0, v0.16b -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i8: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: add v0.16b, v0.16b, v1.16b -; CHECK-SD-DOT-NEXT: addv b0, v0.16b -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i8: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addv b0, v0.16b -; CHECK-GI-BASE-NEXT: addv b1, v1.16b -; CHECK-GI-BASE-NEXT: fmov w8, s0 -; CHECK-GI-BASE-NEXT: fmov w9, s1 -; CHECK-GI-BASE-NEXT: add w8, w9, w8, uxtb -; CHECK-GI-BASE-NEXT: and w0, w8, #0xff -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i8: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addv b0, v0.16b -; CHECK-GI-DOT-NEXT: addv b1, v1.16b -; CHECK-GI-DOT-NEXT: fmov w8, s0 -; CHECK-GI-DOT-NEXT: fmov w9, s1 -; CHECK-GI-DOT-NEXT: add w8, w9, w8, uxtb -; CHECK-GI-DOT-NEXT: and w0, w8, #0xff -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v16i8_v16i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: add v0.16b, v0.16b, v1.16b +; CHECK-SD-NEXT: addv b0, v0.16b +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v16i8_v16i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addv b0, v0.16b +; CHECK-GI-NEXT: addv b1, v1.16b +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w9, w8, uxtb +; CHECK-GI-NEXT: and w0, w8, #0xff +; CHECK-GI-NEXT: ret entry: %z1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x) %z2 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %y) @@ -5400,157 +4121,81 @@ entry: } define i64 @add_pair_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll2 v2.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v3.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v4.4s, v2.4h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v5.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v6.4s, v3.8h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v7.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v3.4s, v3.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v16.2d, v5.4s, v2.4s -; CHECK-SD-BASE-NEXT: uaddl v2.2d, v5.2s, v2.2s -; CHECK-SD-BASE-NEXT: uaddl2 v5.2d, v0.4s, v4.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v4.2s -; CHECK-SD-BASE-NEXT: uaddl2 v4.2d, v7.4s, v6.4s -; CHECK-SD-BASE-NEXT: uaddl v6.2d, v7.2s, v6.2s -; CHECK-SD-BASE-NEXT: uaddl2 v7.2d, v1.4s, v3.4s -; CHECK-SD-BASE-NEXT: uaddl v1.2d, v1.2s, v3.2s -; CHECK-SD-BASE-NEXT: add v3.2d, v5.2d, v16.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v2.2d, v7.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v6.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v3.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll2 v2.8h, v0.16b, #0 -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll2 v3.8h, v1.16b, #0 -; CHECK-SD-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-DOT-NEXT: ushll v4.4s, v2.4h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v5.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v6.4s, v3.8h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v7.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v3.4s, v3.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v16.2d, v5.4s, v2.4s -; CHECK-SD-DOT-NEXT: uaddl v2.2d, v5.2s, v2.2s -; CHECK-SD-DOT-NEXT: uaddl2 v5.2d, v0.4s, v4.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v4.2s -; CHECK-SD-DOT-NEXT: uaddl2 v4.2d, v7.4s, v6.4s -; CHECK-SD-DOT-NEXT: uaddl v6.2d, v7.2s, v6.2s -; CHECK-SD-DOT-NEXT: uaddl2 v7.2d, v1.4s, v3.4s -; CHECK-SD-DOT-NEXT: uaddl v1.2d, v1.2s, v3.2s -; CHECK-SD-DOT-NEXT: add v3.2d, v5.2d, v16.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v2.2d, v7.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v6.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v3.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v4.4s, v2.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v5.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v6.4s, v3.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v3.4s, v3.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v7.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v16.2d, v4.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v17.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v18.2d, v5.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v19.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v20.2d, v6.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v21.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v22.2d, v7.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v23.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v4.2d, v16.2d, v4.4s -; CHECK-GI-BASE-NEXT: uaddw2 v2.2d, v17.2d, v2.4s -; CHECK-GI-BASE-NEXT: uaddw2 v5.2d, v18.2d, v5.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v19.2d, v0.4s -; CHECK-GI-BASE-NEXT: uaddw2 v6.2d, v20.2d, v6.4s -; CHECK-GI-BASE-NEXT: uaddw2 v3.2d, v21.2d, v3.4s -; CHECK-GI-BASE-NEXT: uaddw2 v7.2d, v22.2d, v7.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v23.2d, v1.4s -; CHECK-GI-BASE-NEXT: add v2.2d, v4.2d, v2.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v5.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v3.2d, v6.2d, v3.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v7.2d, v1.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-DOT-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-DOT-NEXT: ushll v4.4s, v2.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v5.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v6.4s, v3.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v3.4s, v3.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v7.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v16.2d, v4.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v17.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v18.2d, v5.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v19.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v20.2d, v6.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v21.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v22.2d, v7.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v23.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v4.2d, v16.2d, v4.4s -; CHECK-GI-DOT-NEXT: uaddw2 v2.2d, v17.2d, v2.4s -; CHECK-GI-DOT-NEXT: uaddw2 v5.2d, v18.2d, v5.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v19.2d, v0.4s -; CHECK-GI-DOT-NEXT: uaddw2 v6.2d, v20.2d, v6.4s -; CHECK-GI-DOT-NEXT: uaddw2 v3.2d, v21.2d, v3.4s -; CHECK-GI-DOT-NEXT: uaddw2 v7.2d, v22.2d, v7.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v23.2d, v1.4s -; CHECK-GI-DOT-NEXT: add v2.2d, v4.2d, v2.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v5.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v3.2d, v6.2d, v3.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v7.2d, v1.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v16i8_v16i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll2 v2.8h, v0.16b, #0 +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-NEXT: ushll v4.4s, v2.4h, #0 +; CHECK-SD-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-SD-NEXT: ushll2 v5.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll2 v6.4s, v3.8h, #0 +; CHECK-SD-NEXT: ushll2 v7.4s, v1.8h, #0 +; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: uaddl2 v16.2d, v5.4s, v2.4s +; CHECK-SD-NEXT: uaddl v2.2d, v5.2s, v2.2s +; CHECK-SD-NEXT: uaddl2 v5.2d, v0.4s, v4.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v4.2s +; CHECK-SD-NEXT: uaddl2 v4.2d, v7.4s, v6.4s +; CHECK-SD-NEXT: uaddl v6.2d, v7.2s, v6.2s +; CHECK-SD-NEXT: uaddl2 v7.2d, v1.4s, v3.4s +; CHECK-SD-NEXT: uaddl v1.2d, v1.2s, v3.2s +; CHECK-SD-NEXT: add v3.2d, v5.2d, v16.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: add v2.2d, v7.2d, v4.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v6.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v3.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v16i8_v16i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: ushll v4.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll v5.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v6.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: ushll v7.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v16.2d, v4.2s, #0 +; CHECK-GI-NEXT: ushll v17.2d, v2.2s, #0 +; CHECK-GI-NEXT: ushll v18.2d, v5.2s, #0 +; CHECK-GI-NEXT: ushll v19.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v20.2d, v6.2s, #0 +; CHECK-GI-NEXT: ushll v21.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll v22.2d, v7.2s, #0 +; CHECK-GI-NEXT: ushll v23.2d, v1.2s, #0 +; CHECK-GI-NEXT: uaddw2 v4.2d, v16.2d, v4.4s +; CHECK-GI-NEXT: uaddw2 v2.2d, v17.2d, v2.4s +; CHECK-GI-NEXT: uaddw2 v5.2d, v18.2d, v5.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v19.2d, v0.4s +; CHECK-GI-NEXT: uaddw2 v6.2d, v20.2d, v6.4s +; CHECK-GI-NEXT: uaddw2 v3.2d, v21.2d, v3.4s +; CHECK-GI-NEXT: uaddw2 v7.2d, v22.2d, v7.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v23.2d, v1.4s +; CHECK-GI-NEXT: add v2.2d, v4.2d, v2.2d +; CHECK-GI-NEXT: add v0.2d, v5.2d, v0.2d +; CHECK-GI-NEXT: add v3.2d, v6.2d, v3.2d +; CHECK-GI-NEXT: add v1.2d, v7.2d, v1.2d +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v3.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <16 x i8> %x to <16 x i64> %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) @@ -5561,157 +4206,81 @@ entry: } define i64 @add_pair_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll2 v2.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v3.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v4.4s, v2.4h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v2.4s, v2.8h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v5.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v6.4s, v3.8h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v7.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v3.4s, v3.4h, #0 -; CHECK-SD-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v16.2d, v5.4s, v2.4s -; CHECK-SD-BASE-NEXT: saddl v2.2d, v5.2s, v2.2s -; CHECK-SD-BASE-NEXT: saddl2 v5.2d, v0.4s, v4.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v4.2s -; CHECK-SD-BASE-NEXT: saddl2 v4.2d, v7.4s, v6.4s -; CHECK-SD-BASE-NEXT: saddl v6.2d, v7.2s, v6.2s -; CHECK-SD-BASE-NEXT: saddl2 v7.2d, v1.4s, v3.4s -; CHECK-SD-BASE-NEXT: saddl v1.2d, v1.2s, v3.2s -; CHECK-SD-BASE-NEXT: add v3.2d, v5.2d, v16.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v2.2d, v7.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v6.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v3.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll2 v2.8h, v0.16b, #0 -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: sshll2 v3.8h, v1.16b, #0 -; CHECK-SD-DOT-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-DOT-NEXT: sshll v4.4s, v2.4h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v2.4s, v2.8h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v5.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v6.4s, v3.8h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v7.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v3.4s, v3.4h, #0 -; CHECK-SD-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v16.2d, v5.4s, v2.4s -; CHECK-SD-DOT-NEXT: saddl v2.2d, v5.2s, v2.2s -; CHECK-SD-DOT-NEXT: saddl2 v5.2d, v0.4s, v4.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v4.2s -; CHECK-SD-DOT-NEXT: saddl2 v4.2d, v7.4s, v6.4s -; CHECK-SD-DOT-NEXT: saddl v6.2d, v7.2s, v6.2s -; CHECK-SD-DOT-NEXT: saddl2 v7.2d, v1.4s, v3.4s -; CHECK-SD-DOT-NEXT: saddl v1.2d, v1.2s, v3.2s -; CHECK-SD-DOT-NEXT: add v3.2d, v5.2d, v16.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v2.2d, v7.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v6.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v3.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v4.4s, v2.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v2.4s, v2.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v5.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v6.4s, v3.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v3.4s, v3.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v7.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v16.2d, v4.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v17.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v18.2d, v5.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v19.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v20.2d, v6.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v21.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v22.2d, v7.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v23.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v4.2d, v16.2d, v4.4s -; CHECK-GI-BASE-NEXT: saddw2 v2.2d, v17.2d, v2.4s -; CHECK-GI-BASE-NEXT: saddw2 v5.2d, v18.2d, v5.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v19.2d, v0.4s -; CHECK-GI-BASE-NEXT: saddw2 v6.2d, v20.2d, v6.4s -; CHECK-GI-BASE-NEXT: saddw2 v3.2d, v21.2d, v3.4s -; CHECK-GI-BASE-NEXT: saddw2 v7.2d, v22.2d, v7.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v23.2d, v1.4s -; CHECK-GI-BASE-NEXT: add v2.2d, v4.2d, v2.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v5.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v3.2d, v6.2d, v3.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v7.2d, v1.2d -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-DOT-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-GI-DOT-NEXT: sshll v4.4s, v2.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v2.4s, v2.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v5.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v6.4s, v3.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v3.4s, v3.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v7.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v16.2d, v4.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v17.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v18.2d, v5.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v19.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v20.2d, v6.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v21.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v22.2d, v7.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v23.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v4.2d, v16.2d, v4.4s -; CHECK-GI-DOT-NEXT: saddw2 v2.2d, v17.2d, v2.4s -; CHECK-GI-DOT-NEXT: saddw2 v5.2d, v18.2d, v5.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v19.2d, v0.4s -; CHECK-GI-DOT-NEXT: saddw2 v6.2d, v20.2d, v6.4s -; CHECK-GI-DOT-NEXT: saddw2 v3.2d, v21.2d, v3.4s -; CHECK-GI-DOT-NEXT: saddw2 v7.2d, v22.2d, v7.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v23.2d, v1.4s -; CHECK-GI-DOT-NEXT: add v2.2d, v4.2d, v2.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v5.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v3.2d, v6.2d, v3.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v7.2d, v1.2d -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v16i8_v16i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll2 v2.8h, v0.16b, #0 +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll2 v3.8h, v1.16b, #0 +; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-NEXT: sshll v4.4s, v2.4h, #0 +; CHECK-SD-NEXT: sshll2 v2.4s, v2.8h, #0 +; CHECK-SD-NEXT: sshll2 v5.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: sshll2 v6.4s, v3.8h, #0 +; CHECK-SD-NEXT: sshll2 v7.4s, v1.8h, #0 +; CHECK-SD-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: saddl2 v16.2d, v5.4s, v2.4s +; CHECK-SD-NEXT: saddl v2.2d, v5.2s, v2.2s +; CHECK-SD-NEXT: saddl2 v5.2d, v0.4s, v4.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v4.2s +; CHECK-SD-NEXT: saddl2 v4.2d, v7.4s, v6.4s +; CHECK-SD-NEXT: saddl v6.2d, v7.2s, v6.2s +; CHECK-SD-NEXT: saddl2 v7.2d, v1.4s, v3.4s +; CHECK-SD-NEXT: saddl v1.2d, v1.2s, v3.2s +; CHECK-SD-NEXT: add v3.2d, v5.2d, v16.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d +; CHECK-SD-NEXT: add v2.2d, v7.2d, v4.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v6.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v3.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v16i8_v16i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v3.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: sshll v4.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll v5.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v6.4s, v3.4h, #0 +; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: sshll v7.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v16.2d, v4.2s, #0 +; CHECK-GI-NEXT: sshll v17.2d, v2.2s, #0 +; CHECK-GI-NEXT: sshll v18.2d, v5.2s, #0 +; CHECK-GI-NEXT: sshll v19.2d, v0.2s, #0 +; CHECK-GI-NEXT: sshll v20.2d, v6.2s, #0 +; CHECK-GI-NEXT: sshll v21.2d, v3.2s, #0 +; CHECK-GI-NEXT: sshll v22.2d, v7.2s, #0 +; CHECK-GI-NEXT: sshll v23.2d, v1.2s, #0 +; CHECK-GI-NEXT: saddw2 v4.2d, v16.2d, v4.4s +; CHECK-GI-NEXT: saddw2 v2.2d, v17.2d, v2.4s +; CHECK-GI-NEXT: saddw2 v5.2d, v18.2d, v5.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v19.2d, v0.4s +; CHECK-GI-NEXT: saddw2 v6.2d, v20.2d, v6.4s +; CHECK-GI-NEXT: saddw2 v3.2d, v21.2d, v3.4s +; CHECK-GI-NEXT: saddw2 v7.2d, v22.2d, v7.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v23.2d, v1.4s +; CHECK-GI-NEXT: add v2.2d, v4.2d, v2.2d +; CHECK-GI-NEXT: add v0.2d, v5.2d, v0.2d +; CHECK-GI-NEXT: add v3.2d, v6.2d, v3.2d +; CHECK-GI-NEXT: add v1.2d, v7.2d, v1.2d +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v3.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <16 x i8> %x to <16 x i64> %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx) @@ -5722,93 +4291,49 @@ entry: } define i64 @add_pair_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v2.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll2 v3.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: uaddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v2.2s -; CHECK-SD-BASE-NEXT: uaddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-BASE-NEXT: uaddl v1.2d, v1.2s, v3.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-DOT-NEXT: ushll2 v2.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll2 v3.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: uaddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v2.2s -; CHECK-SD-DOT-NEXT: uaddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-DOT-NEXT: uaddl v1.2d, v1.2s, v3.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v5.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v7.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: uaddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-BASE-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: uaddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v3.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v5.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v7.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: uaddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: uaddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-DOT-NEXT: uaddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: uaddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i8_v8i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll2 v3.4s, v1.8h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: uaddl2 v4.2d, v0.4s, v2.4s +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v2.2s +; CHECK-SD-NEXT: uaddl2 v2.2d, v1.4s, v3.4s +; CHECK-SD-NEXT: uaddl v1.2d, v1.2s, v3.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i8_v8i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: ushll v5.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: ushll v7.2d, v1.2s, #0 +; CHECK-GI-NEXT: uaddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: uaddw2 v0.2d, v5.2d, v0.4s +; CHECK-GI-NEXT: uaddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: uaddw2 v1.2d, v7.2d, v1.4s +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v3.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <8 x i8> %x to <8 x i64> %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -5819,93 +4344,49 @@ entry: } define i64 @add_pair_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v2.4s, v0.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: sshll2 v3.4s, v1.8h, #0 -; CHECK-SD-BASE-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: saddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-BASE-NEXT: saddl v0.2d, v0.2s, v2.2s -; CHECK-SD-BASE-NEXT: saddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-BASE-NEXT: saddl v1.2d, v1.2s, v3.2s -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-BASE-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-DOT-NEXT: sshll2 v2.4s, v0.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: sshll2 v3.4s, v1.8h, #0 -; CHECK-SD-DOT-NEXT: sshll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: saddl2 v4.2d, v0.4s, v2.4s -; CHECK-SD-DOT-NEXT: saddl v0.2d, v0.2s, v2.2s -; CHECK-SD-DOT-NEXT: saddl2 v2.2d, v1.4s, v3.4s -; CHECK-SD-DOT-NEXT: saddl v1.2d, v1.2s, v3.2s -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v4.2d -; CHECK-SD-DOT-NEXT: add v1.2d, v1.2d, v2.2d -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v5.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-BASE-NEXT: sshll v7.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-BASE-NEXT: saddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-BASE-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-BASE-NEXT: saddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-BASE-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-GI-DOT-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-DOT-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v3.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: sshll v4.2d, v2.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v5.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v6.2d, v3.2s, #0 -; CHECK-GI-DOT-NEXT: sshll v7.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: saddw2 v2.2d, v4.2d, v2.4s -; CHECK-GI-DOT-NEXT: saddw2 v0.2d, v5.2d, v0.4s -; CHECK-GI-DOT-NEXT: saddw2 v3.2d, v6.2d, v3.4s -; CHECK-GI-DOT-NEXT: saddw2 v1.2d, v7.2d, v1.4s -; CHECK-GI-DOT-NEXT: add v0.2d, v2.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v3.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i8_v8i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: sshll2 v3.4s, v1.8h, #0 +; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: saddl2 v4.2d, v0.4s, v2.4s +; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v2.2s +; CHECK-SD-NEXT: saddl2 v2.2d, v1.4s, v3.4s +; CHECK-SD-NEXT: saddl v1.2d, v1.2s, v3.2s +; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d +; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i8_v8i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll v2.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v4.2d, v2.2s, #0 +; CHECK-GI-NEXT: sshll v5.2d, v0.2s, #0 +; CHECK-GI-NEXT: sshll v6.2d, v3.2s, #0 +; CHECK-GI-NEXT: sshll v7.2d, v1.2s, #0 +; CHECK-GI-NEXT: saddw2 v2.2d, v4.2d, v2.4s +; CHECK-GI-NEXT: saddw2 v0.2d, v5.2d, v0.4s +; CHECK-GI-NEXT: saddw2 v3.2d, v6.2d, v3.4s +; CHECK-GI-NEXT: saddw2 v1.2d, v7.2d, v1.4s +; CHECK-GI-NEXT: add v0.2d, v2.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v3.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <8 x i8> %x to <8 x i64> %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx) @@ -5916,73 +4397,39 @@ entry: } define i64 @add_pair_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i8_v4i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: bic v1.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: uaddlp v1.2d, v1.4s -; CHECK-SD-BASE-NEXT: uadalp v1.2d, v0.4s -; CHECK-SD-BASE-NEXT: addp d0, v1.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i8_v4i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: bic v1.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: uaddlp v1.2d, v1.4s -; CHECK-SD-DOT-NEXT: uadalp v1.2d, v0.4s -; CHECK-SD-DOT-NEXT: addp d0, v1.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i8_v4i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: movi v2.2d, #0x000000000000ff -; CHECK-GI-BASE-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-GI-BASE-NEXT: ushll v4.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.2d, v1.4s, #0 -; CHECK-GI-BASE-NEXT: and v3.16b, v3.16b, v2.16b -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-BASE-NEXT: and v4.16b, v4.16b, v2.16b -; CHECK-GI-BASE-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-BASE-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-BASE-NEXT: add v1.2d, v4.2d, v1.2d -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i8_v4i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: movi v2.2d, #0x000000000000ff -; CHECK-GI-DOT-NEXT: ushll v3.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-GI-DOT-NEXT: ushll v4.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.2d, v1.4s, #0 -; CHECK-GI-DOT-NEXT: and v3.16b, v3.16b, v2.16b -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-DOT-NEXT: and v4.16b, v4.16b, v2.16b -; CHECK-GI-DOT-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-DOT-NEXT: add v0.2d, v3.2d, v0.2d -; CHECK-GI-DOT-NEXT: add v1.2d, v4.2d, v1.2d -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i8_v4i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: bic v1.4h, #255, lsl #8 +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddlp v1.2d, v1.4s +; CHECK-SD-NEXT: uadalp v1.2d, v0.4s +; CHECK-SD-NEXT: addp d0, v1.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i8_v4i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: movi v2.2d, #0x000000000000ff +; CHECK-GI-NEXT: ushll v3.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll2 v0.2d, v0.4s, #0 +; CHECK-GI-NEXT: ushll v4.2d, v1.2s, #0 +; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0 +; CHECK-GI-NEXT: and v3.16b, v3.16b, v2.16b +; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-GI-NEXT: and v4.16b, v4.16b, v2.16b +; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b +; CHECK-GI-NEXT: add v0.2d, v3.2d, v0.2d +; CHECK-GI-NEXT: add v1.2d, v4.2d, v1.2d +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <4 x i8> %x to <4 x i64> %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -5993,93 +4440,49 @@ entry: } define i64 @add_pair_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v4i8_v4i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-BASE-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: ushll v3.2d, v1.2s, #0 -; CHECK-SD-BASE-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-SD-BASE-NEXT: ushll2 v1.2d, v1.4s, #0 -; CHECK-SD-BASE-NEXT: shl v2.2d, v2.2d, #56 -; CHECK-SD-BASE-NEXT: shl v3.2d, v3.2d, #56 -; CHECK-SD-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: sshr v2.2d, v2.2d, #56 -; CHECK-SD-BASE-NEXT: sshr v3.2d, v3.2d, #56 -; CHECK-SD-BASE-NEXT: ssra v2.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: ssra v3.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: add v0.2d, v2.2d, v3.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v4i8_v4i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-DOT-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: ushll v3.2d, v1.2s, #0 -; CHECK-SD-DOT-NEXT: ushll2 v0.2d, v0.4s, #0 -; CHECK-SD-DOT-NEXT: ushll2 v1.2d, v1.4s, #0 -; CHECK-SD-DOT-NEXT: shl v2.2d, v2.2d, #56 -; CHECK-SD-DOT-NEXT: shl v3.2d, v3.2d, #56 -; CHECK-SD-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: sshr v2.2d, v2.2d, #56 -; CHECK-SD-DOT-NEXT: sshr v3.2d, v3.2d, #56 -; CHECK-SD-DOT-NEXT: ssra v2.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: ssra v3.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: add v0.2d, v2.2d, v3.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v4i8_v4i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v2.2d, v0.4s, #0 -; CHECK-GI-BASE-NEXT: ushll2 v3.2d, v1.4s, #0 -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: shl v2.2d, v2.2d, #56 -; CHECK-GI-BASE-NEXT: shl v3.2d, v3.2d, #56 -; CHECK-GI-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: sshr v2.2d, v2.2d, #56 -; CHECK-GI-BASE-NEXT: sshr v3.2d, v3.2d, #56 -; CHECK-GI-BASE-NEXT: ssra v2.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: ssra v3.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: addp d0, v2.2d -; CHECK-GI-BASE-NEXT: addp d1, v3.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v4i8_v4i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v2.2d, v0.4s, #0 -; CHECK-GI-DOT-NEXT: ushll2 v3.2d, v1.4s, #0 -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: shl v2.2d, v2.2d, #56 -; CHECK-GI-DOT-NEXT: shl v3.2d, v3.2d, #56 -; CHECK-GI-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: sshr v2.2d, v2.2d, #56 -; CHECK-GI-DOT-NEXT: sshr v3.2d, v3.2d, #56 -; CHECK-GI-DOT-NEXT: ssra v2.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: ssra v3.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: addp d0, v2.2d -; CHECK-GI-DOT-NEXT: addp d1, v3.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v4i8_v4i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: ushll v2.2d, v0.2s, #0 +; CHECK-SD-NEXT: ushll v3.2d, v1.2s, #0 +; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #0 +; CHECK-SD-NEXT: ushll2 v1.2d, v1.4s, #0 +; CHECK-SD-NEXT: shl v2.2d, v2.2d, #56 +; CHECK-SD-NEXT: shl v3.2d, v3.2d, #56 +; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-SD-NEXT: sshr v2.2d, v2.2d, #56 +; CHECK-SD-NEXT: sshr v3.2d, v3.2d, #56 +; CHECK-SD-NEXT: ssra v2.2d, v0.2d, #56 +; CHECK-SD-NEXT: ssra v3.2d, v1.2d, #56 +; CHECK-SD-NEXT: add v0.2d, v2.2d, v3.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v4i8_v4i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v2.2d, v0.4s, #0 +; CHECK-GI-NEXT: ushll2 v3.2d, v1.4s, #0 +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: shl v2.2d, v2.2d, #56 +; CHECK-GI-NEXT: shl v3.2d, v3.2d, #56 +; CHECK-GI-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-GI-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: sshr v2.2d, v2.2d, #56 +; CHECK-GI-NEXT: sshr v3.2d, v3.2d, #56 +; CHECK-GI-NEXT: ssra v2.2d, v0.2d, #56 +; CHECK-GI-NEXT: ssra v3.2d, v1.2d, #56 +; CHECK-GI-NEXT: addp d0, v2.2d +; CHECK-GI-NEXT: addp d1, v3.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <4 x i8> %x to <4 x i64> %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx) @@ -6090,53 +4493,29 @@ entry: } define i64 @add_pair_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i8_v2i64_zext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: movi d2, #0x0000ff000000ff -; CHECK-SD-BASE-NEXT: and v0.8b, v0.8b, v2.8b -; CHECK-SD-BASE-NEXT: and v1.8b, v1.8b, v2.8b -; CHECK-SD-BASE-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v2i8_v2i64_zext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: movi d2, #0x0000ff000000ff -; CHECK-SD-DOT-NEXT: and v0.8b, v0.8b, v2.8b -; CHECK-SD-DOT-NEXT: and v1.8b, v1.8b, v2.8b -; CHECK-SD-DOT-NEXT: uaddl v0.2d, v0.2s, v1.2s -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v2i8_v2i64_zext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: movi v2.2d, #0x000000000000ff -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-BASE-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v2i8_v2i64_zext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: movi v2.2d, #0x000000000000ff -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-GI-DOT-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v2i8_v2i64_zext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff +; CHECK-SD-NEXT: and v0.8b, v0.8b, v2.8b +; CHECK-SD-NEXT: and v1.8b, v1.8b, v2.8b +; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i8_v2i64_zext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v2.2d, #0x000000000000ff +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = zext <2 x i8> %x to <2 x i64> %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -6147,59 +4526,32 @@ entry: } define i64 @add_pair_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i8_v2i64_sext: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-SD-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: sshr v0.2d, v0.2d, #56 -; CHECK-SD-BASE-NEXT: ssra v0.2d, v1.2d, #56 -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v2i8_v2i64_sext: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-SD-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-SD-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: sshr v0.2d, v0.2d, #56 -; CHECK-SD-DOT-NEXT: ssra v0.2d, v1.2d, #56 -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v2i8_v2i64_sext: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-BASE-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-BASE-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: sshr v0.2d, v0.2d, #56 -; CHECK-GI-BASE-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v2i8_v2i64_sext: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v0.2d, v0.2s, #0 -; CHECK-GI-DOT-NEXT: ushll v1.2d, v1.2s, #0 -; CHECK-GI-DOT-NEXT: shl v0.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: sshr v0.2d, v0.2d, #56 -; CHECK-GI-DOT-NEXT: sshr v1.2d, v1.2d, #56 -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v2i8_v2i64_sext: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-SD-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #56 +; CHECK-SD-NEXT: ssra v0.2d, v1.2d, #56 +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i8_v2i64_sext: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #0 +; CHECK-GI-NEXT: shl v0.2d, v0.2d, #56 +; CHECK-GI-NEXT: shl v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #56 +; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #56 +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret entry: %xx = sext <2 x i8> %x to <2 x i64> %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx) @@ -6305,69 +4657,37 @@ entry: } define i32 @add_pair_v8i16_v4i32_double_sext_zext_shuffle(<8 x i16> %ax, <8 x i16> %ay, <8 x i16> %bx, <8 x i16> %by) { -; CHECK-SD-BASE-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle: -; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: uaddlp v1.4s, v1.8h -; CHECK-SD-BASE-NEXT: uaddlp v3.4s, v3.8h -; CHECK-SD-BASE-NEXT: uadalp v1.4s, v0.8h -; CHECK-SD-BASE-NEXT: uadalp v3.4s, v2.8h -; CHECK-SD-BASE-NEXT: add v0.4s, v3.4s, v1.4s -; CHECK-SD-BASE-NEXT: addv s0, v0.4s -; CHECK-SD-BASE-NEXT: fmov w0, s0 -; CHECK-SD-BASE-NEXT: ret -; -; CHECK-SD-DOT-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle: -; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: uaddlp v1.4s, v1.8h -; CHECK-SD-DOT-NEXT: uaddlp v3.4s, v3.8h -; CHECK-SD-DOT-NEXT: uadalp v1.4s, v0.8h -; CHECK-SD-DOT-NEXT: uadalp v3.4s, v2.8h -; CHECK-SD-DOT-NEXT: add v0.4s, v3.4s, v1.4s -; CHECK-SD-DOT-NEXT: addv s0, v0.4s -; CHECK-SD-DOT-NEXT: fmov w0, s0 -; CHECK-SD-DOT-NEXT: ret -; -; CHECK-GI-BASE-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle: -; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: ushll v4.4s, v0.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v5.4s, v1.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v6.4s, v2.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-BASE-NEXT: ushll v7.4s, v3.4h, #0 -; CHECK-GI-BASE-NEXT: ushll2 v3.4s, v3.8h, #0 -; CHECK-GI-BASE-NEXT: add v0.4s, v4.4s, v0.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v5.4s, v1.4s -; CHECK-GI-BASE-NEXT: add v2.4s, v6.4s, v2.4s -; CHECK-GI-BASE-NEXT: add v3.4s, v7.4s, v3.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-BASE-NEXT: add v1.4s, v2.4s, v3.4s -; CHECK-GI-BASE-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: fmov w0, s0 -; CHECK-GI-BASE-NEXT: ret -; -; CHECK-GI-DOT-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle: -; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: ushll v4.4s, v0.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v5.4s, v1.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v6.4s, v2.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-DOT-NEXT: ushll v7.4s, v3.4h, #0 -; CHECK-GI-DOT-NEXT: ushll2 v3.4s, v3.8h, #0 -; CHECK-GI-DOT-NEXT: add v0.4s, v4.4s, v0.4s -; CHECK-GI-DOT-NEXT: add v1.4s, v5.4s, v1.4s -; CHECK-GI-DOT-NEXT: add v2.4s, v6.4s, v2.4s -; CHECK-GI-DOT-NEXT: add v3.4s, v7.4s, v3.4s -; CHECK-GI-DOT-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-DOT-NEXT: add v1.4s, v2.4s, v3.4s -; CHECK-GI-DOT-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-GI-DOT-NEXT: addv s0, v0.4s -; CHECK-GI-DOT-NEXT: fmov w0, s0 -; CHECK-GI-DOT-NEXT: ret +; CHECK-SD-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: uaddlp v1.4s, v1.8h +; CHECK-SD-NEXT: uaddlp v3.4s, v3.8h +; CHECK-SD-NEXT: uadalp v1.4s, v0.8h +; CHECK-SD-NEXT: uadalp v3.4s, v2.8h +; CHECK-SD-NEXT: add v0.4s, v3.4s, v1.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v4.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v5.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v6.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll v7.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: add v0.4s, v4.4s, v0.4s +; CHECK-GI-NEXT: add v1.4s, v5.4s, v1.4s +; CHECK-GI-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-GI-NEXT: add v3.4s, v7.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w0, s0 +; CHECK-GI-NEXT: ret entry: %axx = zext <8 x i16> %ax to <8 x i32> %s1h = shufflevector <8 x i32> %axx, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3> @@ -6393,155 +4713,249 @@ entry: } define i64 @add_pair_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y) { -; CHECK-SD-BASE-LABEL: add_pair_v2i64_v2i64: +; CHECK-SD-LABEL: add_pair_v2i64_v2i64: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: addp d0, v0.2d +; CHECK-SD-NEXT: fmov x0, d0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_pair_v2i64_v2i64: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: addp d0, v0.2d +; CHECK-GI-NEXT: addp d1, v1.2d +; CHECK-GI-NEXT: fmov x8, d0 +; CHECK-GI-NEXT: fmov x9, d1 +; CHECK-GI-NEXT: add x0, x8, x9 +; CHECK-GI-NEXT: ret +entry: + %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x) + %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %y) + %z = add i64 %z1, %z2 + ret i64 %z +} + +define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) { +; CHECK-SD-BASE-LABEL: full: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-BASE-NEXT: addp d0, v0.2d -; CHECK-SD-BASE-NEXT: fmov x0, d0 +; CHECK-SD-BASE-NEXT: ldr d0, [x2] +; CHECK-SD-BASE-NEXT: ldr d1, [x0] +; CHECK-SD-BASE-NEXT: // kill: def $w3 killed $w3 def $x3 +; CHECK-SD-BASE-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-SD-BASE-NEXT: sxtw x8, w3 +; CHECK-SD-BASE-NEXT: sxtw x9, w1 +; CHECK-SD-BASE-NEXT: uabdl v0.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: add x11, x2, x8 +; CHECK-SD-BASE-NEXT: add x10, x0, x9 +; CHECK-SD-BASE-NEXT: ldr d2, [x11] +; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: add x10, x10, x9 +; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: ldr d2, [x11] +; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: add x10, x10, x9 +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: ldr d2, [x11] +; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: add x10, x10, x9 +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: ldr d2, [x11] +; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: add x10, x10, x9 +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: ldr d2, [x11] +; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: add x10, x10, x9 +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: ldr d2, [x11] +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: ldr d2, [x11, x8] +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: ldr d1, [x10, x9] +; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-SD-BASE-NEXT: addv s0, v0.4s +; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; -; CHECK-SD-DOT-LABEL: add_pair_v2i64_v2i64: +; CHECK-SD-DOT-LABEL: full: ; CHECK-SD-DOT: // %bb.0: // %entry -; CHECK-SD-DOT-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-SD-DOT-NEXT: addp d0, v0.2d -; CHECK-SD-DOT-NEXT: fmov x0, d0 +; CHECK-SD-DOT-NEXT: ldr d0, [x0] +; CHECK-SD-DOT-NEXT: ldr d1, [x2] +; CHECK-SD-DOT-NEXT: // kill: def $w3 killed $w3 def $x3 +; CHECK-SD-DOT-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-SD-DOT-NEXT: sxtw x8, w3 +; CHECK-SD-DOT-NEXT: sxtw x9, w1 +; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: movi v3.8b, #1 +; CHECK-SD-DOT-NEXT: uabd v0.8b, v0.8b, v1.8b +; CHECK-SD-DOT-NEXT: add x11, x2, x8 +; CHECK-SD-DOT-NEXT: add x10, x0, x9 +; CHECK-SD-DOT-NEXT: ldr d4, [x11] +; CHECK-SD-DOT-NEXT: add x11, x11, x8 +; CHECK-SD-DOT-NEXT: ldr d1, [x10] +; CHECK-SD-DOT-NEXT: add x10, x10, x9 +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: ldr d1, [x10] +; CHECK-SD-DOT-NEXT: ldr d4, [x11] +; CHECK-SD-DOT-NEXT: add x10, x10, x9 +; CHECK-SD-DOT-NEXT: add x11, x11, x8 +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: ldr d1, [x10] +; CHECK-SD-DOT-NEXT: ldr d4, [x11] +; CHECK-SD-DOT-NEXT: add x10, x10, x9 +; CHECK-SD-DOT-NEXT: add x11, x11, x8 +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: ldr d1, [x10] +; CHECK-SD-DOT-NEXT: ldr d4, [x11] +; CHECK-SD-DOT-NEXT: add x10, x10, x9 +; CHECK-SD-DOT-NEXT: add x11, x11, x8 +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: ldr d1, [x10] +; CHECK-SD-DOT-NEXT: ldr d4, [x11] +; CHECK-SD-DOT-NEXT: add x10, x10, x9 +; CHECK-SD-DOT-NEXT: add x11, x11, x8 +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: ldr d1, [x10] +; CHECK-SD-DOT-NEXT: ldr d4, [x11] +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: ldr d1, [x10, x9] +; CHECK-SD-DOT-NEXT: ldr d4, [x11, x8] +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s +; CHECK-SD-DOT-NEXT: fmov w0, s0 ; CHECK-SD-DOT-NEXT: ret ; -; CHECK-GI-BASE-LABEL: add_pair_v2i64_v2i64: +; CHECK-GI-BASE-LABEL: full: ; CHECK-GI-BASE: // %bb.0: // %entry -; CHECK-GI-BASE-NEXT: addp d0, v0.2d -; CHECK-GI-BASE-NEXT: addp d1, v1.2d -; CHECK-GI-BASE-NEXT: fmov x8, d0 -; CHECK-GI-BASE-NEXT: fmov x9, d1 -; CHECK-GI-BASE-NEXT: add x0, x8, x9 +; CHECK-GI-BASE-NEXT: ldr d0, [x2] +; CHECK-GI-BASE-NEXT: ldr d1, [x0] +; CHECK-GI-BASE-NEXT: // kill: def $w3 killed $w3 def $x3 +; CHECK-GI-BASE-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-GI-BASE-NEXT: sxtw x8, w3 +; CHECK-GI-BASE-NEXT: sxtw x9, w1 +; CHECK-GI-BASE-NEXT: uabdl v0.8h, v1.8b, v0.8b +; CHECK-GI-BASE-NEXT: add x11, x2, x8 +; CHECK-GI-BASE-NEXT: add x10, x0, x9 +; CHECK-GI-BASE-NEXT: ldr d2, [x11] +; CHECK-GI-BASE-NEXT: add x11, x11, x8 +; CHECK-GI-BASE-NEXT: ldr d1, [x10] +; CHECK-GI-BASE-NEXT: add x10, x10, x9 +; CHECK-GI-BASE-NEXT: uaddlp v0.4s, v0.8h +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: ldr d2, [x11] +; CHECK-GI-BASE-NEXT: add x11, x11, x8 +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: ldr d1, [x10] +; CHECK-GI-BASE-NEXT: add x10, x10, x9 +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: ldr d2, [x11] +; CHECK-GI-BASE-NEXT: add x11, x11, x8 +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: ldr d1, [x10] +; CHECK-GI-BASE-NEXT: add x10, x10, x9 +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: ldr d2, [x11] +; CHECK-GI-BASE-NEXT: add x11, x11, x8 +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: ldr d1, [x10] +; CHECK-GI-BASE-NEXT: add x10, x10, x9 +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: ldr d2, [x11] +; CHECK-GI-BASE-NEXT: add x11, x11, x8 +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: ldr d1, [x10] +; CHECK-GI-BASE-NEXT: add x10, x10, x9 +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: ldr d2, [x11] +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: ldr d1, [x10] +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: ldr d2, [x11, x8] +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: ldr d1, [x10, x9] +; CHECK-GI-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b +; CHECK-GI-BASE-NEXT: uadalp v0.4s, v1.8h +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: fmov w0, s0 ; CHECK-GI-BASE-NEXT: ret ; -; CHECK-GI-DOT-LABEL: add_pair_v2i64_v2i64: +; CHECK-GI-DOT-LABEL: full: ; CHECK-GI-DOT: // %bb.0: // %entry -; CHECK-GI-DOT-NEXT: addp d0, v0.2d -; CHECK-GI-DOT-NEXT: addp d1, v1.2d -; CHECK-GI-DOT-NEXT: fmov x8, d0 -; CHECK-GI-DOT-NEXT: fmov x9, d1 -; CHECK-GI-DOT-NEXT: add x0, x8, x9 +; CHECK-GI-DOT-NEXT: ldr d0, [x0] +; CHECK-GI-DOT-NEXT: ldr d1, [x2] +; CHECK-GI-DOT-NEXT: // kill: def $w3 killed $w3 def $x3 +; CHECK-GI-DOT-NEXT: // kill: def $w1 killed $w1 def $x1 +; CHECK-GI-DOT-NEXT: sxtw x8, w3 +; CHECK-GI-DOT-NEXT: sxtw x9, w1 +; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-DOT-NEXT: movi v3.8b, #1 +; CHECK-GI-DOT-NEXT: uabd v0.8b, v0.8b, v1.8b +; CHECK-GI-DOT-NEXT: add x11, x2, x8 +; CHECK-GI-DOT-NEXT: add x10, x0, x9 +; CHECK-GI-DOT-NEXT: ldr d4, [x11] +; CHECK-GI-DOT-NEXT: add x11, x11, x8 +; CHECK-GI-DOT-NEXT: ldr d1, [x10] +; CHECK-GI-DOT-NEXT: add x10, x10, x9 +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: ldr d1, [x10] +; CHECK-GI-DOT-NEXT: ldr d4, [x11] +; CHECK-GI-DOT-NEXT: add x10, x10, x9 +; CHECK-GI-DOT-NEXT: add x11, x11, x8 +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: ldr d1, [x10] +; CHECK-GI-DOT-NEXT: ldr d4, [x11] +; CHECK-GI-DOT-NEXT: add x10, x10, x9 +; CHECK-GI-DOT-NEXT: add x11, x11, x8 +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: ldr d1, [x10] +; CHECK-GI-DOT-NEXT: ldr d4, [x11] +; CHECK-GI-DOT-NEXT: add x10, x10, x9 +; CHECK-GI-DOT-NEXT: add x11, x11, x8 +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: ldr d1, [x10] +; CHECK-GI-DOT-NEXT: ldr d4, [x11] +; CHECK-GI-DOT-NEXT: add x10, x10, x9 +; CHECK-GI-DOT-NEXT: add x11, x11, x8 +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: ldr d1, [x10] +; CHECK-GI-DOT-NEXT: ldr d4, [x11] +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: ldr d1, [x10, x9] +; CHECK-GI-DOT-NEXT: ldr d4, [x11, x8] +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b +; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v3.8b +; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s +; CHECK-GI-DOT-NEXT: fmov w0, s0 ; CHECK-GI-DOT-NEXT: ret entry: - %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x) - %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %y) - %z = add i64 %z1, %z2 - ret i64 %z -} - -define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) { -; CHECK-BASE-LABEL: full: -; CHECK-BASE: // %bb.0: // %entry -; CHECK-BASE-NEXT: ldr d0, [x2] -; CHECK-BASE-NEXT: ldr d1, [x0] -; CHECK-BASE-NEXT: // kill: def $w3 killed $w3 def $x3 -; CHECK-BASE-NEXT: // kill: def $w1 killed $w1 def $x1 -; CHECK-BASE-NEXT: sxtw x8, w3 -; CHECK-BASE-NEXT: sxtw x9, w1 -; CHECK-BASE-NEXT: uabdl v0.8h, v1.8b, v0.8b -; CHECK-BASE-NEXT: add x11, x2, x8 -; CHECK-BASE-NEXT: add x10, x0, x9 -; CHECK-BASE-NEXT: ldr d2, [x11] -; CHECK-BASE-NEXT: add x11, x11, x8 -; CHECK-BASE-NEXT: ldr d1, [x10] -; CHECK-BASE-NEXT: add x10, x10, x9 -; CHECK-BASE-NEXT: uaddlp v0.4s, v0.8h -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: ldr d2, [x11] -; CHECK-BASE-NEXT: add x11, x11, x8 -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: ldr d1, [x10] -; CHECK-BASE-NEXT: add x10, x10, x9 -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: ldr d2, [x11] -; CHECK-BASE-NEXT: add x11, x11, x8 -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: ldr d1, [x10] -; CHECK-BASE-NEXT: add x10, x10, x9 -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: ldr d2, [x11] -; CHECK-BASE-NEXT: add x11, x11, x8 -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: ldr d1, [x10] -; CHECK-BASE-NEXT: add x10, x10, x9 -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: ldr d2, [x11] -; CHECK-BASE-NEXT: add x11, x11, x8 -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: ldr d1, [x10] -; CHECK-BASE-NEXT: add x10, x10, x9 -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: ldr d2, [x11] -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: ldr d1, [x10] -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: ldr d2, [x11, x8] -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: ldr d1, [x10, x9] -; CHECK-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b -; CHECK-BASE-NEXT: uadalp v0.4s, v1.8h -; CHECK-BASE-NEXT: addv s0, v0.4s -; CHECK-BASE-NEXT: fmov w0, s0 -; CHECK-BASE-NEXT: ret -; -; CHECK-DOT-LABEL: full: -; CHECK-DOT: // %bb.0: // %entry -; CHECK-DOT-NEXT: ldr d0, [x0] -; CHECK-DOT-NEXT: ldr d1, [x2] -; CHECK-DOT-NEXT: // kill: def $w3 killed $w3 def $x3 -; CHECK-DOT-NEXT: // kill: def $w1 killed $w1 def $x1 -; CHECK-DOT-NEXT: sxtw x8, w3 -; CHECK-DOT-NEXT: sxtw x9, w1 -; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-DOT-NEXT: movi v3.8b, #1 -; CHECK-DOT-NEXT: uabd v0.8b, v0.8b, v1.8b -; CHECK-DOT-NEXT: add x11, x2, x8 -; CHECK-DOT-NEXT: add x10, x0, x9 -; CHECK-DOT-NEXT: ldr d4, [x11] -; CHECK-DOT-NEXT: add x11, x11, x8 -; CHECK-DOT-NEXT: ldr d1, [x10] -; CHECK-DOT-NEXT: add x10, x10, x9 -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: ldr d1, [x10] -; CHECK-DOT-NEXT: ldr d4, [x11] -; CHECK-DOT-NEXT: add x10, x10, x9 -; CHECK-DOT-NEXT: add x11, x11, x8 -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: ldr d1, [x10] -; CHECK-DOT-NEXT: ldr d4, [x11] -; CHECK-DOT-NEXT: add x10, x10, x9 -; CHECK-DOT-NEXT: add x11, x11, x8 -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: ldr d1, [x10] -; CHECK-DOT-NEXT: ldr d4, [x11] -; CHECK-DOT-NEXT: add x10, x10, x9 -; CHECK-DOT-NEXT: add x11, x11, x8 -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: ldr d1, [x10] -; CHECK-DOT-NEXT: ldr d4, [x11] -; CHECK-DOT-NEXT: add x10, x10, x9 -; CHECK-DOT-NEXT: add x11, x11, x8 -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: ldr d1, [x10] -; CHECK-DOT-NEXT: ldr d4, [x11] -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: ldr d1, [x10, x9] -; CHECK-DOT-NEXT: ldr d4, [x11, x8] -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b -; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v3.8b -; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s -; CHECK-DOT-NEXT: fmov w0, s0 -; CHECK-DOT-NEXT: ret -entry: %idx.ext8 = sext i32 %s2 to i64 %idx.ext = sext i32 %s1 to i64 %0 = load <8 x i8>, ptr %p1, align 1 |