diff options
Diffstat (limited to 'llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll')
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 1860 |
1 files changed, 1705 insertions, 155 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll index a9cffd6..ff77d5cc 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll @@ -1,5 +1,8 @@ -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX90A-VGPR %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX942-VGPR %s declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32) declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32) @@ -10,17 +13,238 @@ declare <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double, double, <4 x doubl declare double @llvm.amdgcn.mfma.f64.4x4x4f64(double, double, double, i32, i32, i32) declare i32 @llvm.amdgcn.workitem.id.x() -; GCN-LABEL: {{^}}test_mfma_f32_32x32x4bf16_1k: -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_32x32x4bf16_1k a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_32x32x4_2b_bf16 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-8: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s16 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s17 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s18 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s19 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s20 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s21 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s22 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s23 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s24 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s25 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s26 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s27 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s28 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s29 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s30 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s31 +; GFX90A-NEXT: v_accvgpr_write_b32 a16, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a17, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a18, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a19, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a20, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a21, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a22, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a23, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a24, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a25, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a26, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a27, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a28, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a29, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a30, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a31, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_32x32x4bf16_1k a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[24:27], s[34:35] offset:96 +; GFX90A-NEXT: global_store_dwordx4 v1, a[28:31], s[34:35] offset:112 +; GFX90A-NEXT: global_store_dwordx4 v1, a[16:19], s[34:35] offset:64 +; GFX90A-NEXT: global_store_dwordx4 v1, a[20:23], s[34:35] offset:80 +; GFX90A-NEXT: global_store_dwordx4 v1, a[8:11], s[34:35] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v1, a[12:15], s[34:35] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[34:35] +; GFX90A-NEXT: global_store_dwordx4 v1, a[4:7], s[34:35] offset:16 +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s16 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s17 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s18 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s19 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s20 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s21 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s22 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s23 +; GFX942-NEXT: v_accvgpr_write_b32 a8, s24 +; GFX942-NEXT: v_accvgpr_write_b32 a9, s25 +; GFX942-NEXT: v_accvgpr_write_b32 a10, s26 +; GFX942-NEXT: v_accvgpr_write_b32 a11, s27 +; GFX942-NEXT: v_accvgpr_write_b32 a12, s28 +; GFX942-NEXT: v_accvgpr_write_b32 a13, s29 +; GFX942-NEXT: v_accvgpr_write_b32 a14, s30 +; GFX942-NEXT: v_accvgpr_write_b32 a15, s31 +; GFX942-NEXT: v_accvgpr_write_b32 a16, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a17, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a18, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a19, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a20, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a21, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a22, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a23, s7 +; GFX942-NEXT: v_accvgpr_write_b32 a24, s8 +; GFX942-NEXT: v_accvgpr_write_b32 a25, s9 +; GFX942-NEXT: v_accvgpr_write_b32 a26, s10 +; GFX942-NEXT: v_accvgpr_write_b32 a27, s11 +; GFX942-NEXT: v_accvgpr_write_b32 a28, s12 +; GFX942-NEXT: v_accvgpr_write_b32 a29, s13 +; GFX942-NEXT: v_accvgpr_write_b32 a30, s14 +; GFX942-NEXT: v_accvgpr_write_b32 a31, s15 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 2 +; GFX942-NEXT: global_store_dwordx4 v1, a[24:27], s[34:35] offset:96 +; GFX942-NEXT: global_store_dwordx4 v1, a[28:31], s[34:35] offset:112 +; GFX942-NEXT: global_store_dwordx4 v1, a[16:19], s[34:35] offset:64 +; GFX942-NEXT: global_store_dwordx4 v1, a[20:23], s[34:35] offset:80 +; GFX942-NEXT: global_store_dwordx4 v1, a[8:11], s[34:35] offset:32 +; GFX942-NEXT: global_store_dwordx4 v1, a[12:15], s[34:35] offset:48 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[34:35] +; GFX942-NEXT: global_store_dwordx4 v1, a[4:7], s[34:35] offset:16 +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v33, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v34, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v35, v33 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v32, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_32x32x4bf16_1k v[0:31], v[34:35], v[32:33], v[0:31] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[24:27], s[34:35] offset:96 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[28:31], s[34:35] offset:112 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[16:19], s[34:35] offset:64 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[20:23], s[34:35] offset:80 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[8:11], s[34:35] offset:32 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[12:15], s[34:35] offset:48 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[0:3], s[34:35] +; GFX90A-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16 +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v33, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v34, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v35, v33 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v32, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, s16 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, s17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, s18 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, s19 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, s20 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, s21 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, s22 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, s23 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, s24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v9, s25 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s26 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s27 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s28 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s29 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v14, s30 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v15, s31 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, s0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, s1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v20, s4 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v21, s5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v22, s6 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v23, s7 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v24, s8 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v25, s9 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v26, s10 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v27, s11 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v28, s12 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v29, s13 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v30, s14 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v31, s15 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[34:35], v[32:33], v[0:31] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[24:27], s[34:35] offset:96 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[28:31], s[34:35] offset:112 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[16:19], s[34:35] offset:64 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[20:23], s[34:35] offset:80 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[8:11], s[34:35] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[12:15], s[34:35] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[0:3], s[34:35] +; GFX942-VGPR-NEXT: global_store_dwordx4 v33, v[4:7], s[34:35] offset:16 +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -30,16 +254,134 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x4bf16_1k: -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_16x16x4bf16_1k a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_16x16x4_4b_bf16 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_16x16x4bf16_1k a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 2 +; GFX942-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX942-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX942-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x4bf16_1k v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -49,16 +391,82 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_4x4x4bf16_1k: -; GCN-DAG: s_load_dwordx4 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_4x4x4bf16_1k [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_4x4x4_16b_bf16 [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[RES]], define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_4x4x4bf16_1k a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 4 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 4 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_4x4x4bf16_1k v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 4 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 4 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -68,16 +476,136 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_32x32x8bf16_1k: -; GCN-DAG: s_load_dwordx16 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-16: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_32x32x8bf16_1k a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_32x32x8_bf16 a[{{[0-9]+:[0-9]+}}], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN-COUNT-4: global_store_dwordx4 v{{[0-9]+}}, a[{{[0-9:]+}}] define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX90A-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX90A-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX90A-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX90A-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX90A-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX90A-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX90A-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_32x32x8bf16_1k a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX90A-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX90A-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-NEXT: v_accvgpr_write_b32 a8, s8 +; GFX942-NEXT: v_accvgpr_write_b32 a9, s9 +; GFX942-NEXT: v_accvgpr_write_b32 a10, s10 +; GFX942-NEXT: v_accvgpr_write_b32 a11, s11 +; GFX942-NEXT: v_accvgpr_write_b32 a12, s12 +; GFX942-NEXT: v_accvgpr_write_b32 a13, s13 +; GFX942-NEXT: v_accvgpr_write_b32 a14, s14 +; GFX942-NEXT: v_accvgpr_write_b32 a15, s15 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_32x32x8_bf16 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 2 +; GFX942-NEXT: global_store_dwordx4 v1, a[12:15], s[16:17] offset:48 +; GFX942-NEXT: global_store_dwordx4 v1, a[8:11], s[16:17] offset:32 +; GFX942-NEXT: global_store_dwordx4 v1, a[4:7], s[16:17] offset:16 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[16:17] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_32x32x8bf16_1k v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v17, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v18, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v19, v17 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v16, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[8:9] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[10:11] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[12:13] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[14:15], s[14:15] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_32x32x8_bf16 v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 2 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[12:15], s[16:17] offset:48 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[8:11], s[16:17] offset:32 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[4:7], s[16:17] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v17, v[0:3], s[16:17] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <16 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -87,16 +615,84 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f32_16x16x16bf16_1k: -; GCN-DAG: s_load_dwordx4 -; GCN-DAG: v_mov_b32_e32 v[[TWO:[0-9]+]], 2 -; GCN-DAG: v_mov_b32_e32 v[[ONE:[0-9]+]], 1 -; GCN-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, s{{[0-9]+}} -; GFX90A: v_mfma_f32_16x16x16bf16_1k [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9]+}}], v[[[TWO]]:{{[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f32_16x16x16_bf16 [[RES:a\[[0-9]+:[0-9]+\]]], v[[[ONE]]:{{[0-9+]}}], v[[[TWO]]:{{[0-9+]}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GCN-NOT: v_accvgpr_read_b32 -; GCN: global_store_dwordx4 v{{[0-9]+}}, [[RES]], define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg) #0 { +; GFX90A-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_mov_b32_e32 v2, 1 +; GFX90A-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-NEXT: v_mov_b32_e32 v0, 2 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f32_16x16x16bf16_1k a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 2 +; GFX90A-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1 +; GFX942-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-NEXT: v_mov_b32_e32 v0, 2 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f32_16x16x16_bf16 a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-NEXT: s_nop 6 +; GFX942-NEXT: global_store_dwordx4 v1, a[0:3], s[6:7] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f32_16x16x16bf16_1k v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 2 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, 1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v5 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, 2 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f32_16x16x16_bf16 v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3 +; GFX942-VGPR-NEXT: s_nop 6 +; GFX942-VGPR-NEXT: global_store_dwordx4 v5, v[0:3], s[6:7] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x float>, ptr addrspace(1) %arg %a = bitcast i64 1 to <4 x i16> @@ -106,13 +702,70 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_4x4x4f64: -; GFX90A: v_mfma_f64_4x4x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX90A: v_mfma_f64_4x4x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_4x4x4_4b_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX942: v_mfma_f64_4x4x4_4b_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx2 define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_4x4x4f64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], 0 +; GFX90A-NEXT: s_nop 3 +; GFX90A-NEXT: v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], a[0:1] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: global_store_dwordx2 v0, a[0:1], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_4x4x4f64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], 0 +; GFX942-NEXT: s_nop 3 +; GFX942-NEXT: v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], a[0:1] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: global_store_dwordx2 v0, a[0:1], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_4x4x4f64: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0 +; GFX90A-VGPR-NEXT: s_nop 3 +; GFX90A-VGPR-NEXT: v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_4x4x4f64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0 +; GFX942-VGPR-NEXT: s_nop 3 +; GFX942-VGPR-NEXT: v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double 0.0, i32 0, i32 0, i32 0) %mai.2 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double %mai.1, i32 1, i32 2, i32 3) @@ -120,13 +773,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64: -; GCN: s_load_dwordx8 -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_mov_b32_e32 v2, s10 +; GFX90A-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v3, s11 +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX90A-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX90A-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX90A-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX90A-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX90A-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[8:9] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b32_e32 v2, s10 +; GFX942-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX942-NEXT: v_mov_b32_e32 v3, s11 +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[12:13] +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_accvgpr_write_b32 a0, s0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, s1 +; GFX942-NEXT: v_accvgpr_write_b32 a2, s2 +; GFX942-NEXT: v_accvgpr_write_b32 a3, s3 +; GFX942-NEXT: v_accvgpr_write_b32 a4, s4 +; GFX942-NEXT: v_accvgpr_write_b32 a5, s5 +; GFX942-NEXT: v_accvgpr_write_b32 a6, s6 +; GFX942-NEXT: v_accvgpr_write_b32 a7, s7 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[8:9] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[8:9] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v10, s10 +; GFX90A-VGPR-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v11, s11 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[12:13], s[12:13] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[10:11], v[8:9], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v10, s10 +; GFX942-VGPR-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v11, s11 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[12:13] +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], s[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[10:11], v[8:9], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[8:9] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9] +; GFX942-VGPR-NEXT: s_endpgm bb: %in.1 = load <4 x double>, ptr addrspace(1) %arg %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %in.1, i32 1, i32 2, i32 3) @@ -134,14 +884,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_0: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 0{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 0 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 0 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> zeroinitializer, i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -149,14 +963,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_neg1: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], -1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], -1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 -1 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -164,14 +1042,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_1: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 1.0{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 1.0{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 1.0 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 1.0 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 1.0 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 1.0 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double 1.0), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -179,14 +1121,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_neg1: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1.0{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1.0{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], -1.0 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], -1.0 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1.0 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1.0 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double -1.0), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -194,14 +1200,78 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64: -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 64{{$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 64{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 64 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 64 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 64 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 64 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 64 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -209,23 +1279,116 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: -; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 0{{$}} -; GCN: v_accvgpr_write_b32 a[[A_HIGH_BITS_0:[0-9]+]], 64 -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_HIGH_BITS_0]] - -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, 64 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, 64 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, 64 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[12:13], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[10:11], v[12:13], v[2:9] +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[10:11], v[12:13], v[2:9] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 64 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[12:13], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[10:11], v[12:13], v[2:9] +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[10:11], v[12:13], v[2:9] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877906944 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -233,23 +1396,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: -; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 64{{$}} -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_LOW_BITS_0]] - -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 64 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 64 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 64 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 64 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877907008 to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -257,23 +1507,110 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: -; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 1.0 -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]] -; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_LOW_BITS_0]] - -; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3 -; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0] -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a0 +; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 0 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], s[2:3] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0] +; GFX942-VGPR-NEXT: v_mov_b32_e32 v8, 0 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 0 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (<2 x float> splat (float 1.0) to double)), i32 0, i32 0, i32 0) %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3) @@ -281,26 +1618,236 @@ bb: ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_imm: -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0x3ff00000 +; GFX90A-NEXT: v_accvgpr_write_b32 a7, v0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: v_mov_b32_e32 v3, s3 +; GFX90A-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX942-NEXT: v_mov_b32_e32 v0, 0x3ff00000 +; GFX942-NEXT: v_accvgpr_write_b32 a7, v0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b32_e32 v2, s2 +; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: v_accvgpr_mov_b32 a1, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, 0x3ff00000 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, 0x3ff00000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, v0 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 0.0, double 0.0, double 0.0, double 1.0>, i32 0, i32 0, i32 0) store <4 x double> %mai.1, ptr addrspace(1) %arg ret void } -; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_lit: -; GCN-DAG: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}} -; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x405ec000 -; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a[{{[0-9]+:[0-9]+}}]{{$}} -; GCN: global_store_dwordx4 -; GCN: global_store_dwordx4 define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %arg, double %a, double %b) #0 { +; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX90A: ; %bb.0: ; %bb +; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-NEXT: v_mov_b32_e32 v0, 0x405ec000 +; GFX90A-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX90A-NEXT: v_accvgpr_write_b32 a1, v0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: v_mov_b32_e32 v3, s3 +; GFX90A-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX90A-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX90A-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-NEXT: s_nop 1 +; GFX90A-NEXT: v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 7 +; GFX90A-NEXT: s_nop 0 +; GFX90A-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX90A-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX90A-NEXT: s_endpgm +; +; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX942: ; %bb.0: ; %bb +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-NEXT: v_mov_b32_e32 v0, 0x405ec000 +; GFX942-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX942-NEXT: v_accvgpr_write_b32 a1, v0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mov_b32_e32 v2, s2 +; GFX942-NEXT: v_mov_b32_e32 v3, s3 +; GFX942-NEXT: v_accvgpr_mov_b32 a2, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a3, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a4, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a5, a1 +; GFX942-NEXT: v_accvgpr_mov_b32 a6, a0 +; GFX942-NEXT: v_accvgpr_mov_b32 a7, a1 +; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7] +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] +; GFX942-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 7 +; GFX942-NEXT: s_nop 0 +; GFX942-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; GFX942-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; GFX942-NEXT: s_endpgm +; +; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX90A-VGPR: ; %bb.0: ; %bb +; GFX90A-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX90A-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v1, 0x405ec000 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX90A-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX90A-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] +; GFX90A-VGPR-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: v_mfma_f64_16x16x4f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 7 +; GFX90A-VGPR-NEXT: s_nop 1 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX90A-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX90A-VGPR-NEXT: s_endpgm +; +; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit: +; GFX942-VGPR: ; %bb.0: ; %bb +; GFX942-VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX942-VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v0, 0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v1, 0x405ec000 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v2, v0 +; GFX942-VGPR-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-VGPR-NEXT: v_mov_b32_e32 v12, s2 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v13, s3 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v3, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v4, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v5, v1 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v6, v0 +; GFX942-VGPR-NEXT: v_mov_b32_e32 v7, v1 +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[8:9], v[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[10:11], s[6:7] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[6:7], v[4:5] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[4:5], v[2:3] +; GFX942-VGPR-NEXT: v_mov_b64_e32 v[2:3], v[0:1] +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: v_mfma_f64_16x16x4_f64 v[2:9], v[12:13], v[10:11], v[2:9] +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 7 +; GFX942-VGPR-NEXT: s_nop 1 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[6:9], s[0:1] offset:16 +; GFX942-VGPR-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1] +; GFX942-VGPR-NEXT: s_endpgm bb: %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 123.0, double 123.0, double 123.0, double 123.0>, i32 0, i32 0, i32 0) store <4 x double> %mai.1, ptr addrspace(1) %arg @@ -308,3 +1855,6 @@ bb: } attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GCN: {{.*}} +; VGPR: {{.*}} |