; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-SDAG,GFX942-VGPRCD-SDAG %s ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-SDAG,GFX950-VGPRCD-SDAG %s ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-GISEL,GFX950-VGPRCD-GISEL %s declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64, i64, <4 x i32>, i32, i32, i32) declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64, i64, <16 x i32>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.bf8(i64, i64, <4 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.fp8(i64, i64, <4 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.bf8(i64, i64, <4 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64, i64, <4 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8(i64, i64, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.fp8(i64, i64, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.bf8(i64, i64, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.fp8(i64, i64, <16 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half>, <8 x half>, <4 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.f16(<4 x half>, <8 x half>, <16 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.bf16(<4 x i16>, <8 x i16>, <4 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.bf16(<4 x i16>, <8 x i16>, <16 x float>, i32, i32, i32) declare <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x64.i8(<2 x i32>, <4 x i32>, <4 x i32>, i32, i32, i32) declare <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x32.i8(<2 x i32>, <4 x i32>, <16 x i32>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.bf8(<2 x i32>, <4 x i32>, <4 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.fp8(<2 x i32>, <4 x i32>, <4 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.bf8(<2 x i32>, <4 x i32>, <4 x float>, i32, i32, i32) declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.fp8(<2 x i32>, <4 x i32>, <4 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.bf8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) define amdgpu_kernel void @test_mfma_i32_16x16x32i8(ptr addrspace(1) %arg) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_i32_16x16x32i8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 5 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_i32_16x16x32i8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 6 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64 4294967298, i64 12884901892, <4 x i32> %in.1, i32 1, i32 2, i32 3) store <4 x i32> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_i32_32x32x16i8(ptr addrspace(1) %arg) #0 { ; GFX942-SDAG-LABEL: test_mfma_i32_32x32x16i8: ; GFX942-SDAG: ; %bb.0: ; %bb ; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-SDAG-NEXT: s_nop 7 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_i32_32x32x16i8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 7 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX950-SDAG-LABEL: test_mfma_i32_32x32x16i8: ; GFX950-SDAG: ; %bb.0: ; %bb ; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-SDAG-NEXT: s_nop 1 ; GFX950-SDAG-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-SDAG-NEXT: s_nop 7 ; GFX950-SDAG-NEXT: s_nop 2 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_i32_32x32x16i8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 7 ; GFX950-GISEL-NEXT: s_nop 2 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64 4294967298, i64 12884901892, <16 x i32> %in.1, i32 1, i32 2, i32 3) store <16 x i32> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_16x16x32_bf8_bf8(ptr addrspace(1) %arg) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_bf8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 5 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_bf8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 6 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.bf8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_16x16x32_bf8_fp8(ptr addrspace(1) %arg) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_fp8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 5 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_fp8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 6 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.fp8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_16x16x32_fp8_bf8(ptr addrspace(1) %arg) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_bf8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 5 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_bf8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 6 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.bf8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_16x16x32_fp8_fp8(ptr addrspace(1) %arg) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_fp8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 5 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_fp8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 6 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_32x32x16_bf8_bf8(ptr addrspace(1) %arg) #0 { ; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_bf8: ; GFX942-SDAG: ; %bb.0: ; %bb ; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-SDAG-NEXT: s_nop 7 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_bf8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 7 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_bf8: ; GFX950-SDAG: ; %bb.0: ; %bb ; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-SDAG-NEXT: s_nop 1 ; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-SDAG-NEXT: s_nop 7 ; GFX950-SDAG-NEXT: s_nop 2 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_bf8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 7 ; GFX950-GISEL-NEXT: s_nop 2 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_32x32x16_bf8_fp8(ptr addrspace(1) %arg) #0 { ; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_fp8: ; GFX942-SDAG: ; %bb.0: ; %bb ; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-SDAG-NEXT: s_nop 7 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_fp8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 7 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_fp8: ; GFX950-SDAG: ; %bb.0: ; %bb ; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-SDAG-NEXT: s_nop 1 ; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-SDAG-NEXT: s_nop 7 ; GFX950-SDAG-NEXT: s_nop 2 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_fp8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 7 ; GFX950-GISEL-NEXT: s_nop 2 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.fp8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_32x32x16_fp8_bf8(ptr addrspace(1) %arg) #0 { ; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_bf8: ; GFX942-SDAG: ; %bb.0: ; %bb ; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-SDAG-NEXT: s_nop 7 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_bf8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 7 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_bf8: ; GFX950-SDAG: ; %bb.0: ; %bb ; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-SDAG-NEXT: s_nop 1 ; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-SDAG-NEXT: s_nop 7 ; GFX950-SDAG-NEXT: s_nop 2 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_bf8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 7 ; GFX950-GISEL-NEXT: s_nop 2 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.bf8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_mfma_f32_32x32x16_fp8_fp8(ptr addrspace(1) %arg) #0 { ; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_fp8: ; GFX942-SDAG: ; %bb.0: ; %bb ; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-SDAG-NEXT: s_nop 7 ; GFX942-SDAG-NEXT: s_nop 1 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-SDAG-NEXT: s_endpgm ; ; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_fp8: ; GFX942-GISEL: ; %bb.0: ; %bb ; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-GISEL-NEXT: s_nop 7 ; GFX942-GISEL-NEXT: s_nop 1 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX942-GISEL-NEXT: s_endpgm ; ; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_fp8: ; GFX950-SDAG: ; %bb.0: ; %bb ; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-SDAG-NEXT: s_nop 1 ; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-SDAG-NEXT: s_nop 7 ; GFX950-SDAG-NEXT: s_nop 2 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-SDAG-NEXT: s_endpgm ; ; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_fp8: ; GFX950-GISEL: ; %bb.0: ; %bb ; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 ; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 ; GFX950-GISEL-NEXT: s_nop 1 ; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 ; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-GISEL-NEXT: s_nop 7 ; GFX950-GISEL-NEXT: s_nop 2 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 ; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 ; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.fp8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_f32_16x16x32_f16(ptr addrspace(1) %arg, <4 x half> %a, <8 x half> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_f16: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_f16: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-LABEL: test_smfmac_f32_16x16x32_f16: ; GFX942-AGPRCD: ; %bb.0: ; %bb ; GFX942-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX942-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-NEXT: s_nop 1 ; GFX942-AGPRCD-NEXT: v_smfmac_f32_16x16x32_f16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-NEXT: s_nop 5 ; GFX942-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] ; GFX942-AGPRCD-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_f16: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_f16: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-LABEL: test_smfmac_f32_16x16x32_f16: ; GFX950-AGPRCD: ; %bb.0: ; %bb ; GFX950-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX950-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-NEXT: s_nop 1 ; GFX950-AGPRCD-NEXT: v_smfmac_f32_16x16x32_f16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-NEXT: s_nop 6 ; GFX950-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] ; GFX950-AGPRCD-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half> %a, <8 x half> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_f32_32x32x16_f16(ptr addrspace(1) %arg, <4 x half> %a, <8 x half> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.f16(<4 x half> %a, <8 x half> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_f32_16x16x32_bf16(ptr addrspace(1) %arg, <4 x i16> %a, <8 x i16> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_bf16: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_bf16: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-LABEL: test_smfmac_f32_16x16x32_bf16: ; GFX942-AGPRCD: ; %bb.0: ; %bb ; GFX942-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX942-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-NEXT: s_nop 1 ; GFX942-AGPRCD-NEXT: v_smfmac_f32_16x16x32_bf16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-NEXT: s_nop 5 ; GFX942-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] ; GFX942-AGPRCD-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_bf16: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_bf16: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-LABEL: test_smfmac_f32_16x16x32_bf16: ; GFX950-AGPRCD: ; %bb.0: ; %bb ; GFX950-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 ; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] ; GFX950-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] ; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] ; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-NEXT: s_nop 1 ; GFX950-AGPRCD-NEXT: v_smfmac_f32_16x16x32_bf16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-NEXT: s_nop 6 ; GFX950-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] ; GFX950-AGPRCD-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.bf16(<4 x i16> %a, <8 x i16> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_f32_32x32x16_bf16(ptr addrspace(1) %arg, <4 x i16> %a, <8 x i16> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.bf16(<4 x i16> %a, <8 x i16> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_16x16x64_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x64.i8(<2 x i32> %a, <4 x i32> %b, <4 x i32> %in.1, i32 %idx, i32 1, i32 2) store <4 x i32> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x32.i8(<2 x i32> %a, <4 x i32> %b, <16 x i32> %in.1, i32 %idx, i32 1, i32 2) store <16 x i32> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.bf8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.fp8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.bf8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 ; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.fp8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) store <4 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.bf8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { ; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX942-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX942-AGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 ; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 ; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 ; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c ; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] ; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 ; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 ; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] ; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 ; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 ; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 ; GFX950-VGPRCD-GISEL-NEXT: s_endpgm ; ; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c ; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 ; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 ; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-SDAG-NEXT: s_endpgm ; ; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: ; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 ; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 ; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c ; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 ; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 ; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] ; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 ; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 ; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 ; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) store <16 x float> %mai.1, ptr addrspace(1) %arg ret void } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX942: {{.*}} ; GFX942-VGPRCD: {{.*}} ; GFX950: {{.*}} ; GFX950-VGPRCD: {{.*}}