diff options
Diffstat (limited to 'llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll')
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll | 4953 |
1 files changed, 4712 insertions, 241 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll index ec4e1cb..beda16c 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll @@ -1,12 +1,9 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -global-isel -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,AGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stress-regalloc=10 -global-isel -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,AGPRCD %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-SDAG,GFX942-VGPRCD-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-VGPRCD,GFX942-GISEL,GFX942-VGPRCD-GISEL %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,VGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -stress-regalloc=10 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,AGPRCD %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -stress-regalloc=10 -global-isel -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL,AGPRCD %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-SDAG,GFX950-VGPRCD-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950,GFX950-VGPRCD,GFX950-GISEL,GFX950-VGPRCD-GISEL %s declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64, i64, <4 x i32>, i32, i32, i32) declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64, i64, <16 x i32>, i32, i32, i32) @@ -33,17 +30,132 @@ declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32>, <4 x i3 declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32>, <4 x i32>, <16 x float>, i32, i32, i32) -; GCN-LABEL: {{^}}test_mfma_i32_16x16x32i8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_i32_16x16x32_i8 a[{{[0-9]+:[0-9]+}}], v[[[TWO]]:[[ONE]]], v[[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_i32_16x16x32_i8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_i32_16x16x32i8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_i32_16x16x32i8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_i32_16x16x32i8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_i32_16x16x32i8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x32.i8(i64 4294967298, i64 12884901892, <4 x i32> %in.1, i32 1, i32 2, i32 3) @@ -51,17 +163,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_i32_32x32x16i8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_i32_32x32x16_i8 a[{{[0-9]+:[0-9]+}}], v[[[TWO]]:[[ONE]]], v[[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_i32_32x32x16_i8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_i32_32x32x16i8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_i32_32x32x16i8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_i32_32x32x16i8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_i32_32x32x16i8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_i32_32x32x16i8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_i32_32x32x16_i8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x16.i8(i64 4294967298, i64 12884901892, <16 x i32> %in.1, i32 1, i32 2, i32 3) @@ -69,17 +318,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_bf8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_bf8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.bf8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -87,17 +451,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_bf8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_bf8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_bf8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_bf8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf8.fp8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -105,17 +584,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_fp8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_fp8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_bf8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.bf8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -123,17 +717,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x32_fp8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_16x16x32_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_16x16x32_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x32_fp8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 5 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 4 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, 3 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-VGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 6 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_mfma_f32_16x16x32_fp8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v4, a[0:3], s[6:7] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 4294967298, i64 12884901892, <4 x float> %in.1, i32 1, i32 2, i32 3) @@ -141,17 +850,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_bf8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_bf8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_bf8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.bf8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -159,17 +1005,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_bf8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_bf8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_bf8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_bf8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_bf8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf8.fp8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -177,17 +1160,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_fp8_bf8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_fp8_bf8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_fp8_bf8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_bf8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_bf8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.bf8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -195,17 +1315,154 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x16_fp8_fp8: -; GFX942-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GFX942-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GFX942-DAG: v_mov_b32_e32 v[[THREE:[0-9]+]], 3 -; GFX942-DAG: v_mov_b32_e32 v[[FOUR:[0-9]+]], 4 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX942: v_mfma_f32_32x32x16_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v{{\[}}[[TWO]]:[[ONE]]], v{{\[}}[[FOUR]]:[[THREE]]], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GISEL: v_mfma_f32_32x32x16_fp8_fp8 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x16_fp8_fp8(ptr addrspace(1) %arg) #0 { +; GFX942-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX942-SDAG: ; %bb.0: ; %bb +; GFX942-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-SDAG-NEXT: s_nop 7 +; GFX942-SDAG-NEXT: s_nop 1 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-SDAG-NEXT: s_endpgm +; +; GFX942-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX942-GISEL: ; %bb.0: ; %bb +; GFX942-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-GISEL-NEXT: s_nop 7 +; GFX942-GISEL-NEXT: s_nop 1 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX942-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX942-GISEL-NEXT: s_endpgm +; +; GFX950-SDAG-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX950-SDAG: ; %bb.0: ; %bb +; GFX950-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-SDAG-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-SDAG-NEXT: s_nop 1 +; GFX950-SDAG-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-SDAG-NEXT: s_nop 7 +; GFX950-SDAG-NEXT: s_nop 2 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-SDAG-NEXT: s_endpgm +; +; GFX950-GISEL-LABEL: test_mfma_f32_32x32x16_fp8_fp8: +; GFX950-GISEL: ; %bb.0: ; %bb +; GFX950-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 2 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 4 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, 3 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX950-GISEL-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX950-GISEL-NEXT: s_nop 1 +; GFX950-GISEL-NEXT: v_mfma_f32_32x32x16_fp8_fp8 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-GISEL-NEXT: s_nop 7 +; GFX950-GISEL-NEXT: s_nop 2 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[16:17] +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[16:17] offset:16 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[16:17] offset:32 +; GFX950-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[16:17] offset:48 +; GFX950-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.fp8.fp8(i64 4294967298, i64 12884901892, <16 x float> %in.1, i32 1, i32 2, i32 3) @@ -213,15 +1470,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_16x16x32_f16: -; GCN: s_load_dwordx4 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x32_f16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_f32_16x16x32_f16(ptr addrspace(1) %arg, <4 x half> %a, <8 x half> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX942-AGPRCD: ; %bb.0: ; %bb +; GFX942-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-NEXT: s_nop 1 +; GFX942-AGPRCD-NEXT: v_smfmac_f32_16x16x32_f16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-NEXT: s_nop 5 +; GFX942-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX942-AGPRCD-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_f16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-LABEL: test_smfmac_f32_16x16x32_f16: +; GFX950-AGPRCD: ; %bb.0: ; %bb +; GFX950-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-NEXT: s_nop 1 +; GFX950-AGPRCD-NEXT: v_smfmac_f32_16x16x32_f16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-NEXT: s_nop 6 +; GFX950-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX950-AGPRCD-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.f16(<4 x half> %a, <8 x half> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -229,18 +1603,278 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_32x32x16_f16: -; GCN: s_load_dwordx16 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x16_f16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_f32_32x32x16_f16(ptr addrspace(1) %arg, <4 x half> %a, <8 x half> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_f16: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_f16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.f16(<4 x half> %a, <8 x half> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -248,15 +1882,132 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_16x16x32_bf16: -; GCN: s_load_dwordx4 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x32_bf16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_f32_16x16x32_bf16(ptr addrspace(1) %arg, <4 x i16> %a, <8 x i16> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX942-AGPRCD: ; %bb.0: ; %bb +; GFX942-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX942-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-NEXT: s_nop 1 +; GFX942-AGPRCD-NEXT: v_smfmac_f32_16x16x32_bf16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-NEXT: s_nop 5 +; GFX942-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX942-AGPRCD-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v7, s6 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v7 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v6, v[8:11], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s6 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x32_bf16 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-LABEL: test_smfmac_f32_16x16x32_bf16: +; GFX950-AGPRCD: ; %bb.0: ; %bb +; GFX950-AGPRCD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[4:5], s[10:11] +; GFX950-AGPRCD-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX950-AGPRCD-NEXT: v_mov_b64_e32 v[2:3], s[14:15] +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-NEXT: s_nop 1 +; GFX950-AGPRCD-NEXT: v_smfmac_f32_16x16x32_bf16 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-NEXT: s_nop 6 +; GFX950-AGPRCD-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX950-AGPRCD-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x32.bf16(<4 x i16> %a, <8 x i16> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -264,18 +2015,278 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_f32_32x32x16_bf16: -; GCN: s_load_dwordx16 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x16_bf16 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_f32_32x32x16_bf16(ptr addrspace(1) %arg, <4 x i16> %a, <8 x i16> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s24 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_f32_32x32x16_bf16: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[26:27] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[24:25], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[28:29] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[30:31] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x16_bf16 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[24:25] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[24:25] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[24:25] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x16.bf16(<4 x i16> %a, <8 x i16> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -283,15 +2294,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_i8: -; GCN: s_load_dwordx4 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_i32_16x16x64_i8 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_i8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <4 x i32> @llvm.amdgcn.smfmac.i32.16x16x64.i8(<2 x i32> %a, <4 x i32> %b, <4 x i32> %in.1, i32 %idx, i32 1, i32 2) @@ -299,18 +2509,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_i8: -; GCN: s_load_dwordx16 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]][[[RLO:[0-9]+]]:{{[0-9]+}}], s[[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_i32_32x32x32_i8 [[CD]][[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_i8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x i32>, ptr addrspace(1) %arg %mai.1 = tail call <16 x i32> @llvm.amdgcn.smfmac.i32.32x32x32.i8(<2 x i32> %a, <4 x i32> %b, <16 x i32> %in.1, i32 %idx, i32 1, i32 2) @@ -318,15 +2820,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_bf8_bf8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_bf8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.bf8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -334,15 +3035,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_bf8_fp8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_bf8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.bf8.fp8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -350,15 +3250,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_fp8_bf8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_fp8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.bf8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -366,15 +3465,214 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_16x16x64_fp8_fp8: -; GCN: s_load_dwordx4 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_16x16x64_fp8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:[[RHI]]] define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 6 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 5 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 5 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v10, s8 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v11, s9 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s10 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s11 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s12 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s13 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s4, s2 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s5, s3 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 6 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s0, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s2 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s3 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_writelane_b32 v7, s0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX950-AGPRCD-GISEL-NEXT: v_readlane_b32 s0, v7, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 a[0:3], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 6 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x64.fp8.fp8(<2 x i32> %a, <4 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -382,18 +3680,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_bf8_bf8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_bf8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.bf8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -401,18 +3991,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_bf8_fp8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_bf8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.bf8.fp8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -420,18 +4302,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_fp8_bf8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_fp8_bf8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.bf8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -439,18 +4613,310 @@ bb: ret void } -; GCN-LABEL: {{^}}test_smfmac_i32_32x32x32_fp8_fp8: -; GCN: s_load_dwordx16 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[{{[0-9:]+}}], 0x0{{$}} -; VGPRCD-DAG: v_mov_b64_e32 [[CD:v]]{{\[}}[[RLO:[0-9]+]]:{{[0-9]+}}], s{{\[}}[[SLO]]:{{[0-9]+}}]{{$}} -; VGPRCD-DAG: v_mov_b64_e32 v[{{[0-9]+}}:[[RHI:[0-9]+]]], s[{{[0-9]+}}:[[SHI]]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 [[CD:a]][[RLO:[0-9]+]], s[[SLO]]{{$}} -; AGPRCD-DAG: v_accvgpr_write_b32 a[[RHI:[0-9]+]], s[[SHI]]{{$}} -; GCN: v_smfmac_f32_32x32x32_fp8_fp8 [[CD]]{{\[}}[[RLO]]:[[RHI]]], {{[av]}}[{{[0-9:]+}}], {{[av]}}[{{[0-9:]+}}], v{{[0-9]+}} cbsz:1 abid:2 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]]{{\[}}[[RLO]]:{{[0-9]+}}], s[{{[0-9:]+}}]{{$}} -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:16 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9:]+}}], s[{{[0-9:]+}}] offset:32 -; GCN-DAG: global_store_dwordx4 v{{[0-9]+}}, [[CD]][{{[0-9]+}}:[[RHI]]], s[{{[0-9:]+}}] offset:48 define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 { +; GFX942-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX942-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX942-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX942-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX942-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX942-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX942-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX942-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX942-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX942-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX942-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX942-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX942-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX942-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX942-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX942-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX942-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX942-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX942-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX942-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX942-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX942-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX942-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX942-AGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-VGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-VGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v22, s16 +; GFX950-VGPRCD-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v23, s17 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v18, s18 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v19, s19 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v20, s20 +; GFX950-VGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v21, s21 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, s22 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-VGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2 +; GFX950-VGPRCD-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-VGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-VGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-VGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24 +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c +; GFX950-VGPRCD-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17] +; GFX950-VGPRCD-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s20, s18 +; GFX950-VGPRCD-GISEL-NEXT: s_mov_b32 s21, s19 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v22, s26 +; GFX950-VGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX950-VGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-VGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2 +; GFX950-VGPRCD-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-VGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25] +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32 +; GFX950-VGPRCD-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48 +; GFX950-VGPRCD-GISEL-NEXT: s_endpgm +; +; GFX950-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-AGPRCD-SDAG: ; %bb.0: ; %bb +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-SDAG-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c +; GFX950-AGPRCD-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v4, s8 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v5, s9 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v3, s13 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v6, s14 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 1 +; GFX950-AGPRCD-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 7 +; GFX950-AGPRCD-SDAG-NEXT: s_nop 2 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-SDAG-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-SDAG-NEXT: s_endpgm +; +; GFX950-AGPRCD-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8: +; GFX950-AGPRCD-GISEL: ; %bb.0: ; %bb +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x2c +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[24:25] +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx16 s[8:23], s[0:1], 0x0 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a0, s8 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a1, s9 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a2, s10 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a3, s11 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a4, s12 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a5, s13 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a6, s14 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a7, s15 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a8, s16 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a9, s17 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a10, s18 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a11, s19 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a12, s20 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a13, s21 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a14, s22 +; GFX950-AGPRCD-GISEL-NEXT: v_accvgpr_write_b32 a15, s23 +; GFX950-AGPRCD-GISEL-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x3c +; GFX950-AGPRCD-GISEL-NEXT: s_load_dword s2, s[4:5], 0x44 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s8, s26 +; GFX950-AGPRCD-GISEL-NEXT: s_mov_b32 s9, s27 +; GFX950-AGPRCD-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v6, s2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; GFX950-AGPRCD-GISEL-NEXT: s_nop 1 +; GFX950-AGPRCD-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 a[0:15], v[4:5], v[0:3], v6 cbsz:1 abid:2 +; GFX950-AGPRCD-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 7 +; GFX950-AGPRCD-GISEL-NEXT: s_nop 2 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; GFX950-AGPRCD-GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; GFX950-AGPRCD-GISEL-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %mai.1 = tail call <16 x float> @llvm.amdgcn.smfmac.f32.32x32x32.fp8.fp8(<2 x i32> %a, <4 x i32> %b, <16 x float> %in.1, i32 %idx, i32 1, i32 2) @@ -459,3 +4925,8 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX942: {{.*}} +; GFX942-VGPRCD: {{.*}} +; GFX950: {{.*}} +; GFX950-VGPRCD: {{.*}} |