; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub
; RUN: llc -mattr=+sve,+bf16            < %s | FileCheck %s
; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s

target triple = "aarch64-unknown-linux-gnu"

;
; FCEIL
;

define <vscale x 2 x bfloat> @frintp_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frintp_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.ceil.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frintp_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frintp_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.ceil.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frintp_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintp_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frintp z1.s, p0/m, z1.s
; CHECK-NEXT:    frintp z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.ceil.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; FFLOOR
;

define <vscale x 2 x bfloat> @frintm_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frintm_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.floor.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frintm_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frintm_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.floor.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frintm_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintm_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frintm z1.s, p0/m, z1.s
; CHECK-NEXT:    frintm z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.floor.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; FNEARBYINT
;

define <vscale x 2 x bfloat> @frinti_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frinti_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.nearbyint.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frinti_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frinti_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.nearbyint.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frinti_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frinti_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frinti z1.s, p0/m, z1.s
; CHECK-NEXT:    frinti z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.nearbyint.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; FRINT
;

define <vscale x 2 x bfloat> @frintx_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frintx_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.rint.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frintx_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frintx_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.rint.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frintx_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintx_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.rint.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; ROUND
;

define <vscale x 2 x bfloat> @frinta_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frinta_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.round.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frinta_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frinta_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.round.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frinta_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frinta_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frinta z1.s, p0/m, z1.s
; CHECK-NEXT:    frinta z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.round.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; ROUNDEVEN
;

define <vscale x 2 x bfloat> @frintn_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frintn_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.roundeven.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frintn_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frintn_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.roundeven.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frintn_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintn_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frintn z1.s, p0/m, z1.s
; CHECK-NEXT:    frintn z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.roundeven.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

;
; FTRUNC
;

define <vscale x 2 x bfloat> @frintz_nxv2bf16(<vscale x 2 x bfloat> %a) {
; CHECK-LABEL: frintz_nxv2bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 2 x bfloat> @llvm.trunc.nxv2bf16(<vscale x 2 x bfloat> %a)
  ret <vscale x 2 x bfloat> %res
}

define <vscale x 4 x bfloat> @frintz_nxv4bf16(<vscale x 4 x bfloat> %a) {
; CHECK-LABEL: frintz_nxv4bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.trunc.nxv4bf16(<vscale x 4 x bfloat> %a)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @frintz_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintz_nxv8bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uunpkhi z1.s, z0.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    lsl z1.s, z1.s, #16
; CHECK-NEXT:    lsl z0.s, z0.s, #16
; CHECK-NEXT:    frintz z1.s, p0/m, z1.s
; CHECK-NEXT:    frintz z0.s, p0/m, z0.s
; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
; CHECK-NEXT:    bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT:    uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.trunc.nxv8bf16(<vscale x 8 x bfloat> %a)
  ret <vscale x 8 x bfloat> %res
}

declare <vscale x 2 x bfloat> @llvm.ceil.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.ceil.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.ceil.nxv8bf16( <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.floor.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.floor.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.floor.nxv8bf16( <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.nearbyint.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.nearbyint.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.nearbyint.nxv8bf16( <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.rint.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.rint.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.rint.nxv8bf16( <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.round.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.round.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.round.nxv8bf16( <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.roundeven.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.roundeven.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.roundeven.nxv8bf16( <vscale x 8 x bfloat>)

declare <vscale x 2 x bfloat> @llvm.trunc.nxv2bf16( <vscale x 2 x bfloat>)
declare <vscale x 4 x bfloat> @llvm.trunc.nxv4bf16( <vscale x 4 x bfloat>)
declare <vscale x 8 x bfloat> @llvm.trunc.nxv8bf16( <vscale x 8 x bfloat>)