[AArch64] Lower extending sitofp using tbl (#92528)

In a similar manner as in https://reviews.llvm.org/D133494
use `TBL` to place bytes in the *upper* part of `i32` elements
and then convert to float using fixed-point `scvtf`, i.e.

    scvtf Vd.4s, Vn.4s, #24

3 files changed, 313 insertions, 3 deletions

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index bb2c76d..aee9935 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -8331,7 +8331,8 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, ModifyDT &ModifiedDT) {
     if (OptimizeNoopCopyExpression(CI, *TLI, *DL))
       return true;
 
-    if ((isa<UIToFPInst>(I) || isa<FPToUIInst>(I) || isa<TruncInst>(I)) &&
+    if ((isa<UIToFPInst>(I) || isa<SIToFPInst>(I) || isa<FPToUIInst>(I) ||
+         isa<TruncInst>(I)) &&
         TLI->optimizeExtendOrTruncateConversion(
             I, LI->getLoopFor(I->getParent()), *TTI))
       return true;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c5c3ef0..35871cc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15869,6 +15869,24 @@ static Value *createTblShuffleForZExt(IRBuilderBase &Builder, Value *Op,
   return Result;
 }
 
+static Value *createTblShuffleForSExt(IRBuilderBase &Builder, Value *Op,
+                                      FixedVectorType *DstTy,
+                                      bool IsLittleEndian) {
+  auto *SrcTy = cast<FixedVectorType>(Op->getType());
+  auto SrcWidth = cast<IntegerType>(SrcTy->getElementType())->getBitWidth();
+  auto DstWidth = cast<IntegerType>(DstTy->getElementType())->getBitWidth();
+
+  SmallVector<int> Mask;
+  if (!createTblShuffleMask(SrcWidth, DstWidth, SrcTy->getNumElements(),
+                            !IsLittleEndian, Mask))
+    return nullptr;
+
+  auto *FirstEltZero = Builder.CreateInsertElement(
+      PoisonValue::get(SrcTy), Builder.getInt8(0), uint64_t(0));
+
+  return Builder.CreateShuffleVector(Op, FirstEltZero, Mask);
+}
+
 static void createTblForTrunc(TruncInst *TI, bool IsLittleEndian) {
   IRBuilder<> Builder(TI);
   SmallVector<Value *> Parts;
@@ -16049,14 +16067,29 @@ bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(
     Value *ZExt = createTblShuffleForZExt(
         Builder, I->getOperand(0), FixedVectorType::getInteger(DstTy),
         FixedVectorType::getInteger(DstTy), Subtarget->isLittleEndian());
-    if (!ZExt)
-      return false;
+    assert(ZExt && "Cannot fail for the i8 to float conversion");
     auto *UI = Builder.CreateUIToFP(ZExt, DstTy);
     I->replaceAllUsesWith(UI);
     I->eraseFromParent();
     return true;
   }
 
+  auto *SIToFP = dyn_cast<SIToFPInst>(I);
+  if (SIToFP && SrcTy->getElementType()->isIntegerTy(8) &&
+      DstTy->getElementType()->isFloatTy()) {
+    IRBuilder<> Builder(I);
+    auto *Shuffle = createTblShuffleForSExt(Builder, I->getOperand(0),
+                                            FixedVectorType::getInteger(DstTy),
+                                            Subtarget->isLittleEndian());
+    assert(Shuffle && "Cannot fail for the i8 to float conversion");
+    auto *Cast = Builder.CreateBitCast(Shuffle, VectorType::getInteger(DstTy));
+    auto *AShr = Builder.CreateAShr(Cast, 24, "", true);
+    auto *SI = Builder.CreateSIToFP(AShr, DstTy);
+    I->replaceAllUsesWith(SI);
+    I->eraseFromParent();
+    return true;
+  }
+
   // Convert 'fptoui <(8|16) x float> to <(8|16) x i8>' to a wide fptoui
   // followed by a truncate lowered to using tbl.4.
   auto *FPToUI = dyn_cast<FPToUIInst>(I);
diff --git a/llvm/test/CodeGen/AArch64/sitofp-to-tbl.ll b/llvm/test/CodeGen/AArch64/sitofp-to-tbl.ll
new file mode 100644
index 0000000..3e708b0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sitofp-to-tbl.ll
@@ -0,0 +1,276 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+; CHECK-LABEL: .LCPI0_0:
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    4
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    5
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    6
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    7
+; CHECK-NEXT:  .LCPI0_1:
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    0
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    1
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    2
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    255
+; CHECK-NEXT:  .byte    3
+
+define void @sitofp_v8i8_to_v8f32(ptr %src, ptr %dst) {
+; CHECK-LABEL: sitofp_v8i8_to_v8f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    adrp x9, .LCPI0_1
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI0_1]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB0_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d2, [x0, x8, lsl #3]
+; CHECK-NEXT:    add x9, x1, x8, lsl #5
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    tbl v3.16b, { v2.16b }, v0.16b
+; CHECK-NEXT:    tbl v2.16b, { v2.16b }, v1.16b
+; CHECK-NEXT:    scvtf v3.4s, v3.4s, #24
+; CHECK-NEXT:    scvtf v2.4s, v2.4s, #24
+; CHECK-NEXT:    stp q2, q3, [x9]
+; CHECK-NEXT:    b.eq .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds <8 x i8>, ptr %src, i64 %iv
+  %l = load <8 x i8>, ptr %gep.src
+  %conv = sitofp <8 x i8> %l to <8 x float>
+  %gep.dst = getelementptr inbounds <8 x float>, ptr %dst, i64 %iv
+  store <8 x float> %conv, ptr %gep.dst
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: .LCPI1_0:
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    12
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    13
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    14
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    15
+; CHECK-NEXT: .LCPI1_1:
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    8
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    9
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    10
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    11
+; CHECK-NEXT: .LCPI1_2:
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    4
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    5
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    6
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    7
+; CHECK-NEXT: .LCPI1_3:
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    0
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    1
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    2
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    255
+; CHECK-NEXT: .byte    3
+
+define void @sitofp_v16i8_to_v16f32(ptr %src, ptr %dst) {
+; CHECK-LABEL: sitofp_v16i8_to_v16f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, .LCPI1_0
+; CHECK-NEXT:    adrp x9, .LCPI1_1
+; CHECK-NEXT:    adrp x10, .LCPI1_2
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT:    adrp x8, .LCPI1_3
+; CHECK-NEXT:    ldr q1, [x9, :lo12:.LCPI1_1]
+; CHECK-NEXT:    ldr q2, [x10, :lo12:.LCPI1_2]
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI1_3]
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB1_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr q4, [x0, x8, lsl #4]
+; CHECK-NEXT:    add x9, x1, x8, lsl #6
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    tbl v5.16b, { v4.16b }, v0.16b
+; CHECK-NEXT:    tbl v6.16b, { v4.16b }, v1.16b
+; CHECK-NEXT:    tbl v7.16b, { v4.16b }, v2.16b
+; CHECK-NEXT:    tbl v4.16b, { v4.16b }, v3.16b
+; CHECK-NEXT:    scvtf v5.4s, v5.4s, #24
+; CHECK-NEXT:    scvtf v6.4s, v6.4s, #24
+; CHECK-NEXT:    scvtf v7.4s, v7.4s, #24
+; CHECK-NEXT:    scvtf v4.4s, v4.4s, #24
+; CHECK-NEXT:    stp q6, q5, [x9, #32]
+; CHECK-NEXT:    stp q4, q7, [x9]
+; CHECK-NEXT:    b.eq .LBB1_1
+; CHECK-NEXT:  // %bb.2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds <16 x i8>, ptr %src, i64 %iv
+  %l = load <16 x i8>, ptr %gep.src
+  %conv = sitofp <16 x i8> %l to <16 x float>
+  %gep.dst = getelementptr inbounds <16 x float>, ptr %dst, i64 %iv
+  store <16 x float> %conv, ptr %gep.dst
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
+; Negative tests, currently we don't convert to f16/bf16 via `tbl`.
+define void @sitofp_v8i8_to_v8f16(ptr %src, ptr %dst) {
+; CHECK-LABEL: sitofp_v8i8_to_v8f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB2_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr d0, [x0, x8, lsl #3]
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-NEXT:    str q1, [x1, x8, lsl #4]
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    b.eq .LBB2_1
+; CHECK-NEXT:  // %bb.2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds <8 x i8>, ptr %src, i64 %iv
+  %l = load <8 x i8>, ptr %gep.src
+  %conv = sitofp <8 x i8> %l to <8 x half>
+  %gep.dst = getelementptr inbounds <8 x half>, ptr %dst, i64 %iv
+  store <8 x half> %conv, ptr %gep.dst
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
+; Negative test, conversion to double with the help of `tbl` not implemented (TODO)
+define void @sitofp_v2i8_to_v2f64(ptr %src, ptr %dst) {
+; CHECK-LABEL: sitofp_v2i8_to_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  .LBB3_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8, lsl #1
+; CHECK-NEXT:    ldrsb w10, [x9]
+; CHECK-NEXT:    ldrsb w9, [x9, #1]
+; CHECK-NEXT:    fmov s0, w10
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    str q0, [x1, x8, lsl #4]
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    b.eq .LBB3_1
+; CHECK-NEXT:  // %bb.2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds <2 x i8>, ptr %src, i64 %iv
+  %l = load <2 x i8>, ptr %gep.src
+  %conv = sitofp <2 x i8> %l to <2 x double>
+  %gep.dst = getelementptr inbounds <2 x double>, ptr %dst, i64 %iv
+  store <2 x double> %conv, ptr %gep.dst
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %loop, label %exit
+
+exit:
+  ret void
+}