diff options
Diffstat (limited to 'llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll')
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll | 512 |
1 files changed, 512 insertions, 0 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll new file mode 100644 index 0000000..1b2eb83 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll @@ -0,0 +1,512 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s + +define bfloat @mixlo_simple(float %src0, float %src1, float %src2) #0 { +; GFX1250-LABEL: mixlo_simple: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2) + %cvt.result = fptrunc float %result to bfloat + ret bfloat %cvt.result +} + +define bfloat @mixlo_simpl_no_flush(float %src0, float %src1, float %src2) { +; GFX1250-LABEL: mixlo_simpl_no_flush: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2) + %cvt.result = fptrunc float %result to bfloat + ret bfloat %cvt.result +} + +define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_bf16lo(bfloat %src0, bfloat %src1, bfloat %src2) #0 { +; GFX1250-LABEL: v_mad_mixlo_bf16_bf16lo_bf16lo_bf16lo: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext bfloat %src0 to float + %src1.ext = fpext bfloat %src1 to float + %src2.ext = fpext bfloat %src2 to float + %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext) + %cvt.result = fptrunc float %result to bfloat + ret bfloat %cvt.result +} + +define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_bf16lo_no_flush(bfloat %src0, bfloat %src1, bfloat %src2) { +; GFX1250-LABEL: v_mad_mixlo_bf16_bf16lo_bf16lo_bf16lo_no_flush: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext bfloat %src0 to float + %src1.ext = fpext bfloat %src1 to float + %src2.ext = fpext bfloat %src2 to float + %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext) + %cvt.result = fptrunc float %result to bfloat + ret bfloat %cvt.result +} + +define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_f32(bfloat %src0, bfloat %src1, float %src2) #0 { +; GFX1250-LABEL: v_mad_mixlo_bf16_bf16lo_bf16lo_f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,0] +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext bfloat %src0 to float + %src1.ext = fpext bfloat %src1 to float + %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2) + %cvt.result = fptrunc float %result to bfloat + ret bfloat %cvt.result +} + +define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_post_cvt(bfloat %src0, bfloat %src1, float %src2) #0 { +; GFX1250-LABEL: v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_post_cvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,0] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext bfloat %src0 to float + %src1.ext = fpext bfloat %src1 to float + %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2) + %cvt.result = fptrunc float %result to bfloat + %max = call bfloat @llvm.maxnum.bf16(bfloat %cvt.result, bfloat 0.0) + %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0) + ret bfloat %clamp +} + +define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_pre_cvt(bfloat %src0, bfloat %src1, float %src2) #0 { +; GFX1250-LABEL: v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_pre_cvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext bfloat %src0 to float + %src1.ext = fpext bfloat %src1 to float + %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2) + %max = call float @llvm.maxnum.f32(float %result, float 0.0) + %clamp = call float @llvm.minnum.f32(float %max, float 1.0) + %cvt.result = fptrunc float %clamp to bfloat + ret bfloat %cvt.result +} + + +define <2 x bfloat> @v_mad_mix_v2f32(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v2f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v0 :: v_dual_lshlrev_b32 v6, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> + %src1.ext = fpext <2 x bfloat> %src1 to <2 x float> + %src2.ext = fpext <2 x bfloat> %src2 to <2 x float> + %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext) + %cvt.result = fptrunc <2 x float> %result to <2 x bfloat> + ret <2 x bfloat> %cvt.result +} + +define <3 x bfloat> @v_mad_mix_v3f32(<3 x bfloat> %src0, <3 x bfloat> %src1, <3 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v3f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] +; GFX1250-NEXT: v_fma_mixlo_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_fma_mixhi_bf16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] +; GFX1250-NEXT: v_mov_b32_e32 v0, v6 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <3 x bfloat> %src0 to <3 x float> + %src1.ext = fpext <3 x bfloat> %src1 to <3 x float> + %src2.ext = fpext <3 x bfloat> %src2 to <3 x float> + %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext) + %cvt.result = fptrunc <3 x float> %result to <3 x bfloat> + ret <3 x bfloat> %cvt.result +} + +define <4 x bfloat> @v_mad_mix_v4f32(<4 x bfloat> %src0, <4 x bfloat> %src1, <4 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v4f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX1250-NEXT: v_lshlrev_b32_e32 v6, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-NEXT: v_and_b32_e32 v9, 0xffff0000, v3 +; GFX1250-NEXT: v_lshlrev_b32_e32 v8, 16, v3 +; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX1250-NEXT: v_and_b32_e32 v11, 0xffff0000, v5 +; GFX1250-NEXT: v_and_b32_e32 v13, 0xffff0000, v4 +; GFX1250-NEXT: v_dual_lshlrev_b32 v12, 16, v4 :: v_dual_lshlrev_b32 v10, 16, v5 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[12:13] +; GFX1250-NEXT: v_pk_fma_f32 v[2:3], v[6:7], v[8:9], v[10:11] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, v3 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <4 x bfloat> %src0 to <4 x float> + %src1.ext = fpext <4 x bfloat> %src1 to <4 x float> + %src2.ext = fpext <4 x bfloat> %src2 to <4 x float> + %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext) + %cvt.result = fptrunc <4 x float> %result to <4 x bfloat> + ret <4 x bfloat> %cvt.result +} + + +define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v2f32_clamp_postcvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v0 :: v_dual_lshlrev_b32 v6, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> + %src1.ext = fpext <2 x bfloat> %src1 to <2 x float> + %src2.ext = fpext <2 x bfloat> %src2 to <2 x float> + %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext) + %cvt.result = fptrunc <2 x float> %result to <2 x bfloat> + %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %cvt.result, <2 x bfloat> zeroinitializer) + %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) + ret <2 x bfloat> %clamp +} + + +define <3 x bfloat> @v_mad_mix_v3f32_clamp_postcvt(<3 x bfloat> %src0, <3 x bfloat> %src1, <3 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v3f32_clamp_postcvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_fma_mixlo_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_fma_mixhi_bf16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v1 clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-NEXT: v_mov_b32_e32 v0, v6 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <3 x bfloat> %src0 to <3 x float> + %src1.ext = fpext <3 x bfloat> %src1 to <3 x float> + %src2.ext = fpext <3 x bfloat> %src2 to <3 x float> + %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext) + %cvt.result = fptrunc <3 x float> %result to <3 x bfloat> + %max = call <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat> %cvt.result, <3 x bfloat> zeroinitializer) + %clamp = call <3 x bfloat> @llvm.minnum.v3bf16(<3 x bfloat> %max, <3 x bfloat> <bfloat 1.0, bfloat 1.0, bfloat 1.0>) + ret <3 x bfloat> %clamp +} + +define <4 x bfloat> @v_mad_mix_v4f32_clamp_postcvt(<4 x bfloat> %src0, <4 x bfloat> %src1, <4 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v4f32_clamp_postcvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v6, 16, v0 :: v_dual_lshlrev_b32 v8, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v9, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v10, 16, v3 +; GFX1250-NEXT: v_and_b32_e32 v11, 0xffff0000, v3 +; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff0000, v4 +; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v4 :: v_dual_lshlrev_b32 v12, 16, v5 +; GFX1250-NEXT: v_and_b32_e32 v13, 0xffff0000, v5 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[6:7], v[0:1], v[2:3] +; GFX1250-NEXT: v_pk_fma_f32 v[2:3], v[8:9], v[10:11], v[12:13] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, v3 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v1 clamp +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <4 x bfloat> %src0 to <4 x float> + %src1.ext = fpext <4 x bfloat> %src1 to <4 x float> + %src2.ext = fpext <4 x bfloat> %src2 to <4 x float> + %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext) + %cvt.result = fptrunc <4 x float> %result to <4 x bfloat> + %max = call <4 x bfloat> @llvm.maxnum.v4bf16(<4 x bfloat> %cvt.result, <4 x bfloat> zeroinitializer) + %clamp = call <4 x bfloat> @llvm.minnum.v4bf16(<4 x bfloat> %max, <4 x bfloat> <bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0>) + ret <4 x bfloat> %clamp +} + +define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v0 :: v_dual_lshlrev_b32 v6, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v0, v0 clamp +; GFX1250-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> + %src1.ext = fpext <2 x bfloat> %src1 to <2 x float> + %src2.ext = fpext <2 x bfloat> %src2 to <2 x float> + %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext) + %cvt.result = fptrunc <2 x float> %result to <2 x bfloat> + %cvt.lo = extractelement <2 x bfloat> %cvt.result, i32 0 + %max.lo = call bfloat @llvm.maxnum.bf16(bfloat %cvt.lo, bfloat 0.0) + %clamp.lo = call bfloat @llvm.minnum.bf16(bfloat %max.lo, bfloat 1.0) + %insert = insertelement <2 x bfloat> %cvt.result, bfloat %clamp.lo, i32 0 + ret <2 x bfloat> %insert +} + +define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v0 :: v_dual_lshlrev_b32 v6, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX1250-NEXT: v_pk_max_num_bf16 v1, v1, v1 clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> + %src1.ext = fpext <2 x bfloat> %src1 to <2 x float> + %src2.ext = fpext <2 x bfloat> %src2 to <2 x float> + %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext) + %cvt.result = fptrunc <2 x float> %result to <2 x bfloat> + %cvt.hi = extractelement <2 x bfloat> %cvt.result, i32 1 + %max.hi = call bfloat @llvm.maxnum.bf16(bfloat %cvt.hi, bfloat 0.0) + %clamp.hi = call bfloat @llvm.minnum.bf16(bfloat %max.hi, bfloat 1.0) + %insert = insertelement <2 x bfloat> %cvt.result, bfloat %clamp.hi, i32 1 + ret <2 x bfloat> %insert +} + + +define <2 x bfloat> @v_mad_mix_v2f32_clamp_precvt(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v2f32_clamp_precvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v4, 16, v0 :: v_dual_lshlrev_b32 v6, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1] +; GFX1250-NEXT: v_max_num_f32_e64 v1, v1, v1 clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_max_num_f32_e64 v0, v0, v0 clamp +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <2 x bfloat> %src0 to <2 x float> + %src1.ext = fpext <2 x bfloat> %src1 to <2 x float> + %src2.ext = fpext <2 x bfloat> %src2 to <2 x float> + %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext) + %max = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %result, <2 x float> zeroinitializer) + %clamp = call <2 x float> @llvm.minnum.v2f32(<2 x float> %max, <2 x float> <float 1.0, float 1.0>) + %cvt.result = fptrunc <2 x float> %clamp to <2 x bfloat> + ret <2 x bfloat> %cvt.result +} + + +define <3 x bfloat> @v_mad_mix_v3f32_clamp_precvt(<3 x bfloat> %src0, <3 x bfloat> %src1, <3 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v3f32_clamp_precvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mix_f32_bf16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: v_fma_mix_f32_bf16 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0 +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <3 x bfloat> %src0 to <3 x float> + %src1.ext = fpext <3 x bfloat> %src1 to <3 x float> + %src2.ext = fpext <3 x bfloat> %src2 to <3 x float> + %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext) + %max = call <3 x float> @llvm.maxnum.v3f32(<3 x float> %result, <3 x float> zeroinitializer) + %clamp = call <3 x float> @llvm.minnum.v3f32(<3 x float> %max, <3 x float> <float 1.0, float 1.0, float 1.0>) + %cvt.result = fptrunc <3 x float> %clamp to <3 x bfloat> + ret <3 x bfloat> %cvt.result +} + +define <4 x bfloat> @v_mad_mix_v4f32_clamp_precvt(<4 x bfloat> %src0, <4 x bfloat> %src1, <4 x bfloat> %src2) #0 { +; GFX1250-LABEL: v_mad_mix_v4f32_clamp_precvt: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v6, 16, v0 :: v_dual_lshlrev_b32 v8, 16, v1 +; GFX1250-NEXT: v_and_b32_e32 v9, 0xffff0000, v1 +; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2 +; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v10, 16, v3 +; GFX1250-NEXT: v_and_b32_e32 v11, 0xffff0000, v3 +; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff0000, v4 +; GFX1250-NEXT: v_and_b32_e32 v13, 0xffff0000, v5 +; GFX1250-NEXT: v_dual_lshlrev_b32 v12, 16, v5 :: v_dual_lshlrev_b32 v2, 16, v4 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_pk_fma_f32 v[4:5], v[8:9], v[10:11], v[12:13] +; GFX1250-NEXT: v_pk_fma_f32 v[0:1], v[6:7], v[0:1], v[2:3] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_max_num_f32_e64 v2, v5, v5 clamp +; GFX1250-NEXT: v_max_num_f32_e64 v1, v1, v1 clamp +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_max_num_f32_e64 v0, v0, v0 clamp +; GFX1250-NEXT: v_max_num_f32_e64 v3, v4, v4 clamp +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v3, v2 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %src0.ext = fpext <4 x bfloat> %src0 to <4 x float> + %src1.ext = fpext <4 x bfloat> %src1 to <4 x float> + %src2.ext = fpext <4 x bfloat> %src2 to <4 x float> + %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext) + %max = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %result, <4 x float> zeroinitializer) + %clamp = call <4 x float> @llvm.minnum.v4f32(<4 x float> %max, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>) + %cvt.result = fptrunc <4 x float> %clamp to <4 x bfloat> + ret <4 x bfloat> %cvt.result +} + +define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 { +; GFX1250-LABEL: mixlo_zext: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] + %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2) + %cvt.result = fptrunc float %result to bfloat + %cvt.result.i16 = bitcast bfloat %cvt.result to i16 + %cvt.result.i32 = zext i16 %cvt.result.i16 to i32 + ret i32 %cvt.result.i32 +} + +define bfloat @mixlo_fptrunc(float %a, float %b) #0 { +; GFX1250-LABEL: mixlo_fptrunc: +; GFX1250: ; %bb.0: ; %.entry +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, 0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] +.entry: + %mul = fmul float %a, %b + %trunc = fptrunc float %mul to bfloat + ret bfloat %trunc +} + +define bfloat @mixlo_fptrunc_no_flush(float %a, float %b) { +; GFX1250-LABEL: mixlo_fptrunc_no_flush: +; GFX1250: ; %bb.0: ; %.entry +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, 0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] +.entry: + %mul = fmul float %a, %b + %trunc = fptrunc float %mul to bfloat + ret bfloat %trunc +} + +define bfloat @mixlo_fptrunc_abs_src_mod(float %a, float %b) #0 { +; GFX1250-LABEL: mixlo_fptrunc_abs_src_mod: +; GFX1250: ; %bb.0: ; %.entry +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, |v0|, v1, 0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] +.entry: + %a.fabs = call float @llvm.fabs.f32(float %a) + %mul = fmul float %a.fabs, %b + %trunc = fptrunc float %mul to bfloat + ret bfloat %trunc +} + +define bfloat @mixlo_fptrunc_neg_src_mod(float %a, float %b) #0 { +; GFX1250-LABEL: mixlo_fptrunc_neg_src_mod: +; GFX1250: ; %bb.0: ; %.entry +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_fma_mixlo_bf16 v0, -v0, v1, 0 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] +.entry: + %a.fneg = fneg float %a + %mul = fmul float %a.fneg, %b + %trunc = fptrunc float %mul to bfloat + ret bfloat %trunc +} + +declare float @llvm.fabs.f32(float) #1 + +declare bfloat @llvm.minnum.bf16(bfloat, bfloat) #1 +declare <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat>, <2 x bfloat>) #1 +declare <3 x bfloat> @llvm.minnum.v3bf16(<3 x bfloat>, <3 x bfloat>) #1 +declare <4 x bfloat> @llvm.minnum.v4bf16(<4 x bfloat>, <4 x bfloat>) #1 + +declare bfloat @llvm.maxnum.bf16(bfloat, bfloat) #1 +declare <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat>, <2 x bfloat>) #1 +declare <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat>, <3 x bfloat>) #1 +declare <4 x bfloat> @llvm.maxnum.v4bf16(<4 x bfloat>, <4 x bfloat>) #1 + +declare float @llvm.minnum.f32(float, float) #1 +declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1 +declare <3 x float> @llvm.minnum.v3f32(<3 x float>, <3 x float>) #1 +declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>) #1 + +declare float @llvm.maxnum.f32(float, float) #1 +declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1 +declare <3 x float> @llvm.maxnum.v3f32(<3 x float>, <3 x float>) #1 +declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>) #1 + +declare float @llvm.fmuladd.f32(float, float, float) #1 +declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #1 +declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>) #1 +declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) #1 + +attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" } +attributes #1 = { nounwind readnone speculatable } |