; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GX900 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX942 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s define <4 x half> @shuffle_v4f16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_23uu: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_23uu: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_23uu: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_234u: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_mov_b32_e32 v0, v6 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_234u: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_mov_b32_e32 v0, v4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v6 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_234u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v6 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_234u: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_u1u3: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_u1u3: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_u1u3: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_u3u1: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_u3u1: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v3 ; GFX942-NEXT: v_mov_b32_e32 v1, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_u3u1: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_u3u1: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_u3uu: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_u3uu: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_u3uu: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_3u6u: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_3u6u: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, s0, v5, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_3u6u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_3u6u: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_3u6u: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_3uu7: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_3uu7: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, s0, v5, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_3uu7: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_3uu7: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_3uu7: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_35u5: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GX900-NEXT: global_load_dword v4, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v4, v5, s4 ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_35u5: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dword v4, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v4, v5, s0 ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_35u5: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v4, v5, 0x7060302 ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_35u5: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_35u5: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_357u: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v4, v6, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_357u: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX942-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v1, s0, v5, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v4, v6, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_357u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_357u: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_357u: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s0, v3, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_0101(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_0123(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_0123: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0123: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_0123: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_0145(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_0145: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0145: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_0145: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_0167: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0167: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_0167: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_2301: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_2301: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v3 ; GFX942-NEXT: v_mov_b32_e32 v1, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2301: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_2301: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_2323(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_2323: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2323: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_2323: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_2345(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_2345: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2345: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_2345: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_2367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_2367: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2367: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_2367: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_4501(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_4501: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off ; GFX9-NEXT: global_load_dword v5, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_4501: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_4501: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-NEXT: global_load_b32 v1, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_4523(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_4523: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off ; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_4523: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_4523: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_4545(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_4545: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_4545: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_4545: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_4567(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_4567: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_4567: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_4567: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_6701(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_6701: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6701: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_6701: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_6723(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_6723: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6723: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_6723: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_6745: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[2:3], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_6745: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[2:3], off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v3 ; GFX942-NEXT: v_mov_b32_e32 v1, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6745: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_6745: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_6767(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_6767: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6767: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4f16_6767: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_2356: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_2356: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v1, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2356: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_2356: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_2356: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_5623: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_5623: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_5623: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_5623: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_5623: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_3456: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v4, v6, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_3456: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v4, v6, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_3456: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_3456: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_5634: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_5634: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_5634: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_5634: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_5734: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_5734: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX942-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_5734: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x7060302 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_5734: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v3 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_5734: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v3, v2, 0x7060302 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4i16_2356: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4i16_2356: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v1, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i16_2356: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4i16_2356: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4i16_2356: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x i16>, ptr addrspace(1) %arg0 %val1 = load <4 x i16>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> ret <4 x i16> %shuffle } define <4 x i16> @shuffle_v4i16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4i16_0167: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i16_0167: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4i16_0167: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x i16>, ptr addrspace(1) %arg0 %val1 = load <4 x i16>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <4 x i32> ret <4 x i16> %shuffle } define <4 x half> @shuffle_v4f16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_0000: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v0, v0, s4 ; GX900-NEXT: v_mov_b32_e32 v1, v0 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_0000: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v0, v0, s0 ; GFX942-NEXT: v_mov_b32_e32 v1, v0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0000: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v0, v0, 0x5040100 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_0000: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_0000: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0x5040100 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> zeroinitializer ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4f16_1010: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_1010: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_1010: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_1010: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_1100: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_mov_b32 s5, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v1, v1, s4 ; GX900-NEXT: v_perm_b32 v1, v1, v1, s5 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_1100: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_mov_b32 s1, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v2, v2, s0 ; GFX942-NEXT: v_perm_b32 v1, v2, v2, s1 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_1100: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v1, v1, 0x7060302 ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_1100: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_1100: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v1, 0x7060302 ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_6161: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0xffff ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4 ; GX900-NEXT: v_mov_b32_e32 v1, v0 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_6161: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0xffff ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_bfi_b32 v0, s0, v5, v4 ; GFX942-NEXT: v_mov_b32_e32 v1, v0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6161: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v4 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_6161: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_6161: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_2333: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_2333: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v1, v0, v0, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2333: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_2333: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_2333: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_6667: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_6667: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v1, v0, v0, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6667: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_6667: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_6667: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v8f16_0101(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8f16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8f16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x half>, ptr addrspace(1) %arg0 %val1 = load <8 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v8f16_0123(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8f16_0123: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_0123: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8f16_0123: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x half>, ptr addrspace(1) %arg0 %val1 = load <8 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v8f16_4589(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8f16_4589: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:8 ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_4589: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:8 ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8f16_4589: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:8 ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x half>, ptr addrspace(1) %arg0 %val1 = load <8 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v8f16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8f16_10_11_2_3: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_10_11_2_3: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8f16_10_11_2_3: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x half>, ptr addrspace(1) %arg0 %val1 = load <8 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v8f16_13_14_2_3: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v8f16_13_14_2_3: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off offset:8 ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_13_14_2_3: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v8f16_13_14_2_3: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v8f16_13_14_2_3: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x half>, ptr addrspace(1) %arg0 %val1 = load <8 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v3f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v3f16_0122: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v1, v1, v1, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v3f16_0122: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v1, v1, v1, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v3f16_0122: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v3f16_0122: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v3f16_0122: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <3 x half>, ptr addrspace(1) %arg0 %val1 = load <3 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <3 x half> %val0, <3 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v2f16_0122: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v1, v0, v0, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v2f16_0122: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v1, v0, v0, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v2f16_0122: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v2f16_0122: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <2 x half>, ptr addrspace(1) %arg0 %val1 = load <2 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <2 x half> %val0, <2 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v6f16_452367: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v6, v1 ; GX900-NEXT: v_mov_b32_e32 v5, v0 ; GX900-NEXT: v_mov_b32_e32 v4, v3 ; GX900-NEXT: v_mov_b32_e32 v3, v2 ; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off ; GX900-NEXT: global_load_dword v7, v[3:4], off ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v2, v7 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v6f16_452367: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx3 v[4:6], v[0:1], off ; GFX942-NEXT: global_load_dword v4, v[2:3], off ; GFX942-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GFX942-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_mov_b32_e32 v0, v6 ; GFX942-NEXT: v_mov_b32_e32 v1, v5 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v2, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v6f16_452367: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v6, v1 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: v_mov_b32_e32 v4, v3 ; GFX10-NEXT: v_mov_b32_e32 v3, v2 ; GFX10-NEXT: global_load_dwordx3 v[0:2], v[5:6], off ; GFX10-NEXT: global_load_dword v7, v[3:4], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v2, v7 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v6f16_452367: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 ; GFX11-NEXT: global_load_b96 v[0:2], v[0:1], off ; GFX11-NEXT: global_load_b32 v3, v[3:4], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v2, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <6 x half>, ptr addrspace(1) %arg0 %val1 = load <6 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> ret <6 x half> %shuffle } define amdgpu_kernel void @fma_shuffle_v2f16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) { ; GX900-LABEL: fma_shuffle_v2f16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GX900-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10 ; GX900-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GX900-NEXT: s_waitcnt lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] ; GX900-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] ; GX900-NEXT: global_load_dwordx2 v[4:5], v6, s[4:5] ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] ; GX900-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] ; GX900-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] ; GX900-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] ; GX900-NEXT: global_store_dwordx2 v6, v[0:1], s[4:5] ; GX900-NEXT: s_endpgm ; ; GFX942-LABEL: fma_shuffle_v2f16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 ; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX942-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] ; GFX942-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] ; GFX942-NEXT: global_load_dwordx2 v[4:5], v6, s[6:7] ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] ; GFX942-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] ; GFX942-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] ; GFX942-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] ; GFX942-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7] ; GFX942-NEXT: s_endpgm ; ; GFX10-LABEL: fma_shuffle_v2f16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_clause 0x2 ; GFX10-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] ; GFX10-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] ; GFX10-NEXT: global_load_dwordx2 v[4:5], v6, s[4:5] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] ; GFX10-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] ; GFX10-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] ; GFX10-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] ; GFX10-NEXT: global_store_dwordx2 v6, v[0:1], s[4:5] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: fma_shuffle_v2f16: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 ; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_clause 0x2 ; GFX11-NEXT: global_load_b64 v[0:1], v6, s[0:1] ; GFX11-NEXT: global_load_b64 v[2:3], v6, s[2:3] ; GFX11-NEXT: global_load_b64 v[4:5], v6, s[4:5] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] ; GFX11-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] ; GFX11-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] ; GFX11-NEXT: global_store_b64 v6, v[0:1], s[4:5] ; GFX11-NEXT: s_endpgm entry: %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() %tmp12 = zext i32 %tmp1 to i64 %arrayidx = getelementptr inbounds <4 x half>, ptr addrspace(1) %A, i64 %tmp12 %tmp14 = load <4 x half>, ptr addrspace(1) %arrayidx, align 8 %arrayidx1 = getelementptr inbounds <4 x half>, ptr addrspace(1) %B, i64 %tmp12 %tmp15 = load <4 x half>, ptr addrspace(1) %arrayidx1, align 8 %arrayidx2 = getelementptr inbounds <4 x half>, ptr addrspace(1) %C, i64 %tmp12 %tmp16 = load <4 x half>, ptr addrspace(1) %arrayidx2, align 8 %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> zeroinitializer %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> poison, <2 x i32> %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> poison, <2 x i32> %tmp20 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp17, <2 x half> %tmp18, <2 x half> %tmp19) %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> poison, <2 x i32> %tmp23 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp21, <2 x half> %tmp22, <2 x half> %tmp20) %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> poison, <4 x i32> %tmp25 = shufflevector <4 x half> %tmp24, <4 x half> %tmp16, <4 x i32> %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> poison, <2 x i32> %tmp28 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp26, <2 x half> %tmp18, <2 x half> %tmp27) %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> %tmp30 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp29, <2 x half> %tmp22, <2 x half> %tmp28) %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> poison, <4 x i32> %tmp32 = shufflevector <4 x half> %tmp25, <4 x half> %tmp31, <4 x i32> store <4 x half> %tmp32, ptr addrspace(1) %arrayidx2, align 8 ret void } define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4f16_0456: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GX900-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4f16_0456: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v6, v4, s0 ; GFX942-NEXT: v_alignbit_b32 v1, v7, v6, 16 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0456: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x5040100 ; GFX10-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4f16_0456: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4f16_0456: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> ret <4 x half> %shuffle } define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in, ptr addrspace(1) %out) { ; GX900-LABEL: shuffle_scalar_load_v8i32_0123: ; GX900: ; %bb.0: ; GX900-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GX900-NEXT: v_mov_b32_e32 v4, 0 ; GX900-NEXT: s_waitcnt lgkmcnt(0) ; GX900-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 ; GX900-NEXT: s_waitcnt lgkmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, s4 ; GX900-NEXT: v_mov_b32_e32 v1, s5 ; GX900-NEXT: v_mov_b32_e32 v2, s6 ; GX900-NEXT: v_mov_b32_e32 v3, s7 ; GX900-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] ; GX900-NEXT: s_endpgm ; ; GFX942-LABEL: shuffle_scalar_load_v8i32_0123: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX942-NEXT: v_mov_b32_e32 v4, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5] ; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7] ; GFX942-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] ; GFX942-NEXT: s_endpgm ; ; GFX10-LABEL: shuffle_scalar_load_v8i32_0123: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, s4 ; GFX10-NEXT: v_mov_b32_e32 v1, s5 ; GFX10-NEXT: v_mov_b32_e32 v2, s6 ; GFX10-NEXT: v_mov_b32_e32 v3, s7 ; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: shuffle_scalar_load_v8i32_0123: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 ; GFX11-NEXT: v_mov_b32_e32 v4, 0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7 ; GFX11-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6 ; GFX11-NEXT: global_store_b128 v4, v[0:3], s[2:3] ; GFX11-NEXT: s_endpgm %ld8 = load <8 x i32>, ptr addrspace(4) %in, align 16 %id = shufflevector <8 x i32> %ld8, <8 x i32> poison, <4 x i32> store <4 x i32> %id, ptr addrspace(1) %out, align 8 ret void } define <2 x half> @low16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: low16bits_v2f16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: low16bits_v2f16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16bits_v2f16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: low16bits_v2f16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: low16bits_v2f16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x half>, ptr addrspace(1) %x0, align 4 %1 = load <2 x half>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> ret <2 x half> %vy1.2.vec.insert } define <2 x half> @hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: hi16bits_v2f16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: hi16bits_v2f16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: hi16bits_v2f16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: hi16bits_v2f16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: hi16bits_v2f16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x half>, ptr addrspace(1) %x0, align 4 %1 = load <2 x half>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> ret <2 x half> %vy1.2.vec.insert } define <2 x half> @low16hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: low16hi16bits_v2f16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0xffff ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_bfi_b32 v0, s4, v4, v5 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: low16hi16bits_v2f16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0xffff ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_bfi_b32 v0, s0, v4, v5 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16hi16bits_v2f16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: low16hi16bits_v2f16: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x half>, ptr addrspace(1) %x0, align 4 %1 = load <2 x half>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> ret <2 x half> %vy1.2.vec.insert } define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GFX9-LABEL: hi16low16bits_v2bf16: ; GFX9: ; %bb.0: ; %entry ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: hi16low16bits_v2bf16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: hi16low16bits_v2bf16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: hi16low16bits_v2bf16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x half>, ptr addrspace(1) %x0, align 4 %1 = load <2 x half>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> ret <2 x half> %vy1.2.vec.insert } define <2 x i16> @i16_low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: i16_low16bits: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: i16_low16bits: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_low16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: i16_low16bits: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: i16_low16bits: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4 %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> ret <2 x i16> %vy1.2.vec.insert } define <2 x i16> @i16_low16hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: i16_low16hi16bits: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0xffff ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_bfi_b32 v0, s4, v4, v5 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: i16_low16hi16bits: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0xffff ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_bfi_b32 v0, s0, v4, v5 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_low16hi16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: i16_low16hi16bits: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4 %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> ret <2 x i16> %vy1.2.vec.insert } define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GFX9-LABEL: i16_hi16low16bits: ; GFX9: ; %bb.0: ; %entry ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_hi16low16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: i16_hi16low16bits: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: i16_hi16low16bits: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4 %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> ret <2 x i16> %vy1.2.vec.insert } define <2 x i16> @i16_hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: i16_hi16bits: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: i16_hi16bits: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_hi16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: i16_hi16bits: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: i16_hi16bits: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4 %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> ret <2 x i16> %vy1.2.vec.insert } define <2 x i16> @v2i16_hi16bits(ptr addrspace(1) %x0) { ; GFX9-LABEL: v2i16_hi16bits: ; GFX9: ; %bb.0: ; %entry ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v2i16_hi16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v2i16_hi16bits: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %load0 = load <2 x i16>, ptr addrspace(1) %x0, align 4 %insert1 = insertelement <2 x i16> poison, i16 0, i32 0 %insert2 = insertelement <2 x i16> %insert1, i16 0, i32 1 %vec.ret = shufflevector <2 x i16> %insert2, <2 x i16> %load0, <2 x i32> ret <2 x i16> %vec.ret } define <2 x half> @v2half_hi16bits(ptr addrspace(1) %x0) { ; GFX9-LABEL: v2half_hi16bits: ; GFX9: ; %bb.0: ; %entry ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v2half_hi16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v2half_hi16bits: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %load0 = load <2 x half>, ptr addrspace(1) %x0, align 4 %insert1 = insertelement <2 x half> poison, half 0.0, i32 0 %insert2 = insertelement <2 x half> %insert1, half 0.0, i32 1 %vec.ret = shufflevector <2 x half> %insert2, <2 x half> %load0, <2 x i32> ret <2 x half> %vec.ret } define void @shuffle_v8f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v8f16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8f16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x half>, ptr addrspace(1) %arg0 %val1 = load <4 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <8 x i32> store <8 x half> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v16f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v16f16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16f16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v16f16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x half>, ptr addrspace(1) %arg0 %val1 = load <8 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x half> %val0, <8 x half> %val1, <16 x i32> store <16 x half> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v32f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v32f16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32f16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX10-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(3) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX10-NEXT: s_waitcnt vmcnt(2) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v32f16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16 ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[14:17], v[0:1], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(3) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32 ; GFX11-NEXT: s_waitcnt vmcnt(2) ; GFX11-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[14:17], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <16 x half>, ptr addrspace(1) %arg0 %val1 = load <16 x half>, ptr addrspace(1) %arg1 %shuffle = shufflevector <16 x half> %val0, <16 x half> %val1, <32 x i32> store <32 x half> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v8i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v8i16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8i16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8i16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x i16>, ptr addrspace(1) %arg0 %val1 = load <4 x i16>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x i16> %val0, <4 x i16> %val1, <8 x i32> store <8 x i16> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v16i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v16i16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16i16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v16i16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x i16>, ptr addrspace(1) %arg0 %val1 = load <8 x i16>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x i16> %val0, <8 x i16> %val1, <16 x i32> store <16 x i16> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v32i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v32i16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32i16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX10-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(3) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX10-NEXT: s_waitcnt vmcnt(2) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v32i16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16 ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[14:17], v[0:1], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(3) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32 ; GFX11-NEXT: s_waitcnt vmcnt(2) ; GFX11-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[14:17], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <16 x i16>, ptr addrspace(1) %arg0 %val1 = load <16 x i16>, ptr addrspace(1) %arg1 %shuffle = shufflevector <16 x i16> %val0, <16 x i16> %val1, <32 x i32> store <32 x i16> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v4i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GX900-LABEL: shuffle_v4i8_concat: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_ushort v0, v[0:1], off ; GX900-NEXT: s_nop 0 ; GX900-NEXT: global_load_short_d16_hi v0, v[2:3], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: global_store_dword v[4:5], v0, off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4i8_concat: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_ushort v6, v[0:1], off ; GFX942-NEXT: global_load_ushort v7, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v7, v6, s0 ; GFX942-NEXT: global_store_dword v[4:5], v0, off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i8_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_ushort v0, v[0:1], off ; GFX10-NEXT: global_load_short_d16_hi v0, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dword v[4:5], v0, off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4i8_concat: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_d16_hi_b16 v0, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: global_store_b32 v[4:5], v0, off ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4i8_concat: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_u16 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_d16_hi_b16 v0, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: global_store_b32 v[4:5], v0, off ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <2 x i8>, ptr addrspace(1) %arg0 %val1 = load <2 x i8>, ptr addrspace(1) %arg1 %shuffle = shufflevector <2 x i8> %val0, <2 x i8> %val1, <4 x i32> store <4 x i8> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v8i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v8i8_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v6, v[0:1], off ; GFX9-NEXT: global_load_dword v7, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: global_store_dwordx2 v[4:5], v[6:7], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8i8_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v6, v[0:1], off ; GFX10-NEXT: global_load_dword v7, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx2 v[4:5], v[6:7], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8i8_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b64 v[4:5], v[0:1], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x i8>, ptr addrspace(1) %arg0 %val1 = load <4 x i8>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x i8> %val0, <4 x i8> %val1, <8 x i32> store <8 x i8> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v16i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v16i8_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16i8_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v16i8_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x i8>, ptr addrspace(1) %arg0 %val1 = load <8 x i8>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x i8> %val0, <8 x i8> %val1, <16 x i32> store <16 x i8> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v32i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v32i8_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32i8_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v32i8_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <16 x i8>, ptr addrspace(1) %arg0 %val1 = load <16 x i8>, ptr addrspace(1) %arg1 %shuffle = shufflevector <16 x i8> %val0, <16 x i8> %val1, <32 x i32> store <32 x i8> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v4i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v4i32_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i32_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4i32_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <2 x i32>, ptr addrspace(1) %arg0 %val1 = load <2 x i32>, ptr addrspace(1) %arg1 %shuffle = shufflevector <2 x i32> %val0, <2 x i32> %val1, <4 x i32> store <4 x i32> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v8i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v8i32_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8i32_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8i32_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x i32>, ptr addrspace(1) %arg0 %val1 = load <4 x i32>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x i32> %val0, <4 x i32> %val1, <8 x i32> store <8 x i32> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v16i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v16i32_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16i32_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX10-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(3) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX10-NEXT: s_waitcnt vmcnt(2) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v16i32_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16 ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[14:17], v[0:1], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(3) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32 ; GFX11-NEXT: s_waitcnt vmcnt(2) ; GFX11-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[14:17], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x i32>, ptr addrspace(1) %arg0 %val1 = load <8 x i32>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x i32> %val0, <8 x i32> %val1, <16 x i32> store <16 x i32> %shuffle, ptr addrspace(1) %out ret void } define <4 x bfloat> @shuffle_v4bf16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_23uu: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_23uu: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_23uu: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_234u: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_mov_b32_e32 v0, v6 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_234u: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_mov_b32_e32 v0, v4 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v6 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_234u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v6 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_234u: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_u1u3: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_u1u3: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_u1u3: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_u3u1: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_u3u1: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v3 ; GFX942-NEXT: v_mov_b32_e32 v1, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_u3u1: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_u3u1: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_u3uu: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_u3uu: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_u3uu: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_3u6u: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_3u6u: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, s0, v5, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_3u6u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_3u6u: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_3u6u: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_3uu7: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_3uu7: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, s0, v5, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_3uu7: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_3uu7: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_3uu7: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_35u5: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GX900-NEXT: global_load_dword v4, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v4, v5, s4 ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_35u5: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX942-NEXT: global_load_dword v4, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v4, v5, s0 ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_35u5: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v4, v5, 0x7060302 ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_35u5: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_35u5: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_357u: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v4, v6, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_357u: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX942-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v1, s0, v5, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v4, v6, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_357u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_357u: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_357u: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, s0, v3, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_0123(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_0123: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0123: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0123: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_0145(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_0145: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0145: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0145: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_0167: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0167: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0167: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_2301: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_2301: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v3 ; GFX942-NEXT: v_mov_b32_e32 v1, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2301: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_2301: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_2323(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_2323: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2323: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_2323: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_2345(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_2345: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2345: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_2345: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_2367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_2367: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2367: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_2367: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_4501(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_4501: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off ; GFX9-NEXT: global_load_dword v5, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4501: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4501: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-NEXT: global_load_b32 v1, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_4523(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_4523: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off ; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4523: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4523: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_4545(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_4545: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4545: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4545: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_4567(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_4567: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4567: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4567: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_6701(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_6701: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6701: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_6701: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_6723(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_6723: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6723: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_6723: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_6745: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[2:3], off ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_6745: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[2:3], off ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v3 ; GFX942-NEXT: v_mov_b32_e32 v1, v2 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6745: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_6745: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_6767(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_6767: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6767: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_6767: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_2356: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v0, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_2356: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v1, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v0, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2356: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_2356: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_2356: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_5623: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_5623: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_5623: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_5623: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_5623: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_3456: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v4, v6, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_3456: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v4, v6, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_3456: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_3456: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_5634: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_5634: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_5634: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_5634: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_5734: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_5734: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX942-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_5734: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x7060302 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v1, v4, v6, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_5734: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v3 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_5734: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v3, v2, 0x7060302 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_0000: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v0, v0, s4 ; GX900-NEXT: v_mov_b32_e32 v1, v0 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_0000: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v0, v0, s0 ; GFX942-NEXT: v_mov_b32_e32 v1, v0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0000: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v0, v0, 0x5040100 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_0000: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_0000: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0x5040100 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> zeroinitializer ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v4bf16_1010: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_1010: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_1010: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_1010: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_1100: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_mov_b32 s5, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v1, v1, s4 ; GX900-NEXT: v_perm_b32 v1, v1, v1, s5 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_1100: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[2:3], v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_mov_b32 s1, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v2, v2, s0 ; GFX942-NEXT: v_perm_b32 v1, v2, v2, s1 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_1100: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v1, v1, 0x7060302 ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_1100: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_1100: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v1, 0x7060302 ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_6161: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0xffff ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4 ; GX900-NEXT: v_mov_b32_e32 v1, v0 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_6161: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0xffff ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_bfi_b32 v0, s0, v5, v4 ; GFX942-NEXT: v_mov_b32_e32 v1, v0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6161: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v4 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_6161: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_6161: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_2333: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_2333: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v1, v0, v0, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2333: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_2333: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_2333: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_6667: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_6667: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v1, v0, v0, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6667: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_6667: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_6667: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v8bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8bf16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v8bf16_0123(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8bf16_0123: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_0123: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_0123: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v8bf16_4589(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8bf16_4589: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:8 ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_4589: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:8 ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_4589: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:8 ; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v8bf16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v8bf16_10_11_2_3: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_10_11_2_3: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_10_11_2_3: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v8bf16_13_14_2_3: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 ; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v1, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v8bf16_13_14_2_3: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off offset:8 ; GFX942-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_alignbit_b32 v0, v7, v6, 16 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v1, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_13_14_2_3: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_alignbit_b32 v0, v6, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v8bf16_13_14_2_3: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8 ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v8bf16_13_14_2_3: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8 ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v3, v2, 16 ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v3bf16_0122: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v1, v1, v1, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v3bf16_0122: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v1, v1, v1, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v3bf16_0122: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v3bf16_0122: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v3bf16_0122: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <3 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <3 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <3 x bfloat> %val0, <3 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GFX9-LABEL: shuffle_v2bf16_0122: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v1, v0, v0, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v2bf16_0122: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v1, v0, v0, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v2bf16_0122: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v2bf16_0122: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v0, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <2 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <2 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <2 x bfloat> %val0, <2 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v6bf16_452367: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v6, v1 ; GX900-NEXT: v_mov_b32_e32 v5, v0 ; GX900-NEXT: v_mov_b32_e32 v4, v3 ; GX900-NEXT: v_mov_b32_e32 v3, v2 ; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off ; GX900-NEXT: global_load_dword v7, v[3:4], off ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_mov_b32_e32 v0, v2 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_mov_b32_e32 v2, v7 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v6bf16_452367: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx3 v[4:6], v[0:1], off ; GFX942-NEXT: global_load_dword v4, v[2:3], off ; GFX942-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GFX942-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_mov_b32_e32 v0, v6 ; GFX942-NEXT: v_mov_b32_e32 v1, v5 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_mov_b32_e32 v2, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v6bf16_452367: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v6, v1 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: v_mov_b32_e32 v4, v3 ; GFX10-NEXT: v_mov_b32_e32 v3, v2 ; GFX10-NEXT: global_load_dwordx3 v[0:2], v[5:6], off ; GFX10-NEXT: global_load_dword v7, v[3:4], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v2, v7 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v6bf16_452367: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 ; GFX11-NEXT: global_load_b96 v[0:2], v[0:1], off ; GFX11-NEXT: global_load_b32 v3, v[3:4], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mov_b32_e32 v2, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <6 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <6 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <6 x bfloat> %val0, <6 x bfloat> %val1, <6 x i32> ret <6 x bfloat> %shuffle } define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) { ; GX900-LABEL: fma_shuffle_v2bf16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10 ; GX900-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0 ; GX900-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GX900-NEXT: s_movk_i32 s2, 0x7fff ; GX900-NEXT: s_mov_b32 s3, 0x7060302 ; GX900-NEXT: s_waitcnt lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1] ; GX900-NEXT: global_load_dwordx2 v[3:4], v0, s[4:5] ; GX900-NEXT: global_load_dwordx2 v[5:6], v0, s[6:7] ; GX900-NEXT: s_waitcnt vmcnt(2) ; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 ; GX900-NEXT: s_waitcnt vmcnt(1) ; GX900-NEXT: v_lshlrev_b32_e32 v8, 16, v3 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_and_b32_e32 v9, 0xffff0000, v5 ; GX900-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GX900-NEXT: v_lshlrev_b32_e32 v5, 16, v5 ; GX900-NEXT: v_and_b32_e32 v11, 0xffff0000, v2 ; GX900-NEXT: v_lshlrev_b32_e32 v12, 16, v4 ; GX900-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GX900-NEXT: v_fma_f32 v7, v8, v9, v7 ; GX900-NEXT: v_fma_f32 v1, v8, v5, v1 ; GX900-NEXT: v_fma_f32 v2, v12, v5, v2 ; GX900-NEXT: v_bfe_u32 v5, v7, 16, 1 ; GX900-NEXT: v_fma_f32 v8, v12, v9, v11 ; GX900-NEXT: v_or_b32_e32 v9, 0x400000, v7 ; GX900-NEXT: v_bfe_u32 v11, v1, 16, 1 ; GX900-NEXT: v_add3_u32 v5, v5, v7, s2 ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 ; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v1 ; GX900-NEXT: v_bfe_u32 v13, v8, 16, 1 ; GX900-NEXT: v_add3_u32 v11, v11, v1, s2 ; GX900-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 ; GX900-NEXT: v_or_b32_e32 v14, 0x400000, v8 ; GX900-NEXT: v_bfe_u32 v15, v2, 16, 1 ; GX900-NEXT: v_add3_u32 v13, v13, v8, s2 ; GX900-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v8, v8 ; GX900-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GX900-NEXT: v_lshlrev_b32_e32 v10, 16, v6 ; GX900-NEXT: v_or_b32_e32 v16, 0x400000, v2 ; GX900-NEXT: v_add3_u32 v15, v15, v2, s2 ; GX900-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 ; GX900-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GX900-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GX900-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc ; GX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GX900-NEXT: v_fma_f32 v1, v3, v10, v1 ; GX900-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GX900-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GX900-NEXT: v_fma_f32 v3, v3, v6, v5 ; GX900-NEXT: v_bfe_u32 v5, v1, 16, 1 ; GX900-NEXT: v_fma_f32 v2, v4, v10, v2 ; GX900-NEXT: v_fma_f32 v4, v4, v6, v7 ; GX900-NEXT: v_or_b32_e32 v6, 0x400000, v1 ; GX900-NEXT: v_bfe_u32 v7, v3, 16, 1 ; GX900-NEXT: v_add3_u32 v5, v5, v1, s2 ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 ; GX900-NEXT: v_or_b32_e32 v8, 0x400000, v3 ; GX900-NEXT: v_bfe_u32 v9, v2, 16, 1 ; GX900-NEXT: v_add3_u32 v7, v7, v3, s2 ; GX900-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 ; GX900-NEXT: v_or_b32_e32 v10, 0x400000, v2 ; GX900-NEXT: v_bfe_u32 v11, v4, 16, 1 ; GX900-NEXT: v_add3_u32 v9, v9, v2, s2 ; GX900-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 ; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v4 ; GX900-NEXT: v_add3_u32 v11, v11, v4, s2 ; GX900-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc ; GX900-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 ; GX900-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc ; GX900-NEXT: v_perm_b32 v2, v4, v2, s3 ; GX900-NEXT: v_perm_b32 v1, v3, v1, s3 ; GX900-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1] ; GX900-NEXT: s_endpgm ; ; GFX942-LABEL: fma_shuffle_v2bf16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0 ; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 ; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX942-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX942-NEXT: s_movk_i32 s2, 0x7fff ; GFX942-NEXT: s_mov_b32 s3, 0x7060302 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[0:1], v6, s[8:9] ; GFX942-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1] ; GFX942-NEXT: global_load_dwordx2 v[4:5], v6, s[10:11] ; GFX942-NEXT: s_waitcnt vmcnt(2) ; GFX942-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX942-NEXT: s_waitcnt vmcnt(1) ; GFX942-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 ; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v4 ; GFX942-NEXT: v_lshlrev_b32_e32 v11, 16, v1 ; GFX942-NEXT: v_and_b32_e32 v12, 0xffff0000, v3 ; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX942-NEXT: v_fmac_f32_e32 v8, v7, v9 ; GFX942-NEXT: v_fmac_f32_e32 v2, v7, v4 ; GFX942-NEXT: v_fmac_f32_e32 v3, v11, v4 ; GFX942-NEXT: v_bfe_u32 v4, v8, 16, 1 ; GFX942-NEXT: v_fmac_f32_e32 v12, v11, v9 ; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v8 ; GFX942-NEXT: v_bfe_u32 v9, v2, 16, 1 ; GFX942-NEXT: v_add3_u32 v4, v4, v8, s2 ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v8, v8 ; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v2 ; GFX942-NEXT: v_bfe_u32 v13, v12, 16, 1 ; GFX942-NEXT: v_add3_u32 v9, v9, v2, s2 ; GFX942-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 ; GFX942-NEXT: v_or_b32_e32 v14, 0x400000, v12 ; GFX942-NEXT: v_bfe_u32 v15, v3, 16, 1 ; GFX942-NEXT: v_add3_u32 v13, v13, v12, s2 ; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v12, v12 ; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX942-NEXT: v_or_b32_e32 v16, 0x400000, v3 ; GFX942-NEXT: v_add3_u32 v15, v15, v3, s2 ; GFX942-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 ; GFX942-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX942-NEXT: v_cndmask_b32_e32 v3, v15, v16, vcc ; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX942-NEXT: v_fmac_f32_e32 v2, v0, v10 ; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX942-NEXT: v_fmac_f32_e32 v4, v0, v5 ; GFX942-NEXT: v_bfe_u32 v0, v2, 16, 1 ; GFX942-NEXT: v_fmac_f32_e32 v3, v1, v10 ; GFX942-NEXT: v_fmac_f32_e32 v7, v1, v5 ; GFX942-NEXT: v_or_b32_e32 v1, 0x400000, v2 ; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 1 ; GFX942-NEXT: v_add3_u32 v0, v0, v2, s2 ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 ; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4 ; GFX942-NEXT: v_bfe_u32 v9, v3, 16, 1 ; GFX942-NEXT: v_add3_u32 v5, v5, v4, s2 ; GFX942-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 ; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v3 ; GFX942-NEXT: v_bfe_u32 v11, v7, 16, 1 ; GFX942-NEXT: v_add3_u32 v9, v9, v3, s2 ; GFX942-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 ; GFX942-NEXT: v_or_b32_e32 v12, 0x400000, v7 ; GFX942-NEXT: v_add3_u32 v11, v11, v7, s2 ; GFX942-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc ; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 ; GFX942-NEXT: v_perm_b32 v0, v2, v0, s3 ; GFX942-NEXT: s_nop 0 ; GFX942-NEXT: v_cndmask_b32_e32 v3, v11, v12, vcc ; GFX942-NEXT: v_perm_b32 v1, v3, v1, s3 ; GFX942-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1] ; GFX942-NEXT: s_endpgm ; ; GFX10-LABEL: fma_shuffle_v2bf16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_clause 0x2 ; GFX10-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] ; GFX10-NEXT: global_load_dwordx2 v[2:3], v6, s[4:5] ; GFX10-NEXT: global_load_dwordx2 v[4:5], v6, s[6:7] ; GFX10-NEXT: s_waitcnt vmcnt(2) ; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v2 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4 ; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v1 ; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_fmac_f32_e32 v7, v8, v9 ; GFX10-NEXT: v_fmac_f32_e32 v0, v8, v4 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_fmac_f32_e32 v11, v12, v9 ; GFX10-NEXT: v_fmac_f32_e32 v1, v12, v4 ; GFX10-NEXT: v_bfe_u32 v4, v7, 16, 1 ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v7 ; GFX10-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v0 ; GFX10-NEXT: v_add3_u32 v4, v4, v7, 0x7fff ; GFX10-NEXT: v_bfe_u32 v15, v1, 16, 1 ; GFX10-NEXT: v_add3_u32 v9, v9, v0, 0x7fff ; GFX10-NEXT: v_bfe_u32 v13, v11, 16, 1 ; GFX10-NEXT: v_or_b32_e32 v16, 0x400000, v1 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX10-NEXT: v_add3_u32 v15, v15, v1, 0x7fff ; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX10-NEXT: v_or_b32_e32 v14, 0x400000, v11 ; GFX10-NEXT: v_add3_u32 v13, v13, v11, 0x7fff ; GFX10-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 ; GFX10-NEXT: v_fmac_f32_e32 v4, v2, v5 ; GFX10-NEXT: v_fmac_f32_e32 v0, v2, v10 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo ; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4 ; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1 ; GFX10-NEXT: v_fmac_f32_e32 v1, v3, v10 ; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff ; GFX10-NEXT: v_bfe_u32 v9, v1, 16, 1 ; GFX10-NEXT: v_fmac_f32_e32 v7, v3, v5 ; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0 ; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v1 ; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 1 ; GFX10-NEXT: v_add3_u32 v9, v9, v1, 0x7fff ; GFX10-NEXT: v_bfe_u32 v11, v7, 16, 1 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v7 ; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff ; GFX10-NEXT: v_add3_u32 v11, v11, v7, 0x7fff ; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo ; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x7060302 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo ; GFX10-NEXT: v_perm_b32 v0, v3, v0, 0x7060302 ; GFX10-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; ; GFX11-TRUE16-LABEL: fma_shuffle_v2bf16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 ; GFX11-TRUE16-NEXT: s_load_b128 s[4:7], s[4:5], 0x0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v6, s[0:1] ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v6, s[4:5] ; GFX11-TRUE16-NEXT: global_load_b64 v[4:5], v6, s[6:7] ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_lshlrev_b32 v8, 16, v2 ; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_and_b32 v0, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v8, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v4, v2, v5 ; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v11, v12, v9 ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11 ; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v2, v10 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v7, v3, v5 ; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v5, v8 :: v_dual_fmac_f32 v1, v3, v10 ; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v1 ; GFX11-TRUE16-NEXT: global_store_b64 v6, v[0:1], s[0:1] ; GFX11-TRUE16-NEXT: s_endpgm ; ; GFX11-FAKE16-LABEL: fma_shuffle_v2bf16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_clause 0x1 ; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 ; GFX11-FAKE16-NEXT: s_load_b128 s[4:7], s[4:5], 0x0 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 3, v0 ; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-FAKE16-NEXT: s_clause 0x2 ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v6, s[0:1] ; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v6, s[4:5] ; GFX11-FAKE16-NEXT: global_load_b64 v[4:5], v6, s[6:7] ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 ; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1 ; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_lshlrev_b32 v8, 16, v2 ; GFX11-FAKE16-NEXT: v_bfe_u32 v15, v1, 16, 1 ; GFX11-FAKE16-NEXT: v_or_b32_e32 v16, 0x400000, v1 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff ; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 ; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_lshlrev_b32 v0, 16, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v8, v4 ; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v7, 16, 1 ; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v7 ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v4, v2, v5 ; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v11, v12, v9 ; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 ; GFX11-FAKE16-NEXT: v_bfe_u32 v13, v11, 16, 1 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff ; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, 0x400000, v11 ; GFX11-FAKE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v1, v3, v10 ; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v1, 16, 1 ; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v7, v3, v5 ; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff ; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v7, 16, 1 ; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7 ; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v2, v10 ; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 ; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff ; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 ; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo ; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0x7060302 ; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v8, vcc_lo ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v3, v0, 0x7060302 ; GFX11-FAKE16-NEXT: global_store_b64 v6, v[0:1], s[0:1] ; GFX11-FAKE16-NEXT: s_endpgm entry: %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() %tmp12 = zext i32 %tmp1 to i64 %arrayidx = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %A, i64 %tmp12 %tmp14 = load <4 x bfloat>, ptr addrspace(1) %arrayidx, align 8 %arrayidx1 = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %B, i64 %tmp12 %tmp15 = load <4 x bfloat>, ptr addrspace(1) %arrayidx1, align 8 %arrayidx2 = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %C, i64 %tmp12 %tmp16 = load <4 x bfloat>, ptr addrspace(1) %arrayidx2, align 8 %tmp17 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> zeroinitializer %tmp18 = shufflevector <4 x bfloat> %tmp15, <4 x bfloat> poison, <2 x i32> %tmp19 = shufflevector <4 x bfloat> %tmp16, <4 x bfloat> poison, <2 x i32> %tmp20 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp17, <2 x bfloat> %tmp18, <2 x bfloat> %tmp19) %tmp21 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> %tmp22 = shufflevector <4 x bfloat> %tmp15, <4 x bfloat> poison, <2 x i32> %tmp23 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp21, <2 x bfloat> %tmp22, <2 x bfloat> %tmp20) %tmp24 = shufflevector <2 x bfloat> %tmp23, <2 x bfloat> poison, <4 x i32> %tmp25 = shufflevector <4 x bfloat> %tmp24, <4 x bfloat> %tmp16, <4 x i32> %tmp26 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> %tmp27 = shufflevector <4 x bfloat> %tmp25, <4 x bfloat> poison, <2 x i32> %tmp28 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp26, <2 x bfloat> %tmp18, <2 x bfloat> %tmp27) %tmp29 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> %tmp30 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp29, <2 x bfloat> %tmp22, <2 x bfloat> %tmp28) %tmp31 = shufflevector <2 x bfloat> %tmp30, <2 x bfloat> poison, <4 x i32> %tmp32 = shufflevector <4 x bfloat> %tmp25, <4 x bfloat> %tmp31, <4 x i32> store <4 x bfloat> %tmp32, ptr addrspace(1) %arrayidx2, align 8 ret void } define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { ; GX900-LABEL: shuffle_v4bf16_0456: ; GX900: ; %bb.0: ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GX900-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: shuffle_v4bf16_0456: ; GFX942: ; %bb.0: ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GFX942-NEXT: global_load_dwordx2 v[6:7], v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v6, v4, s0 ; GFX942-NEXT: v_alignbit_b32 v1, v7, v6, 16 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0456: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x5040100 ; GFX10-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: shuffle_v4bf16_0456: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: shuffle_v4bf16_0456: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> ret <4 x bfloat> %shuffle } define <2 x bfloat> @low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: low16bits: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x5040100 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: low16bits: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x5040100 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: low16bits: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: low16bits: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4 %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> ret <2 x bfloat> %vy1.2.vec.insert } define <2 x bfloat> @hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: hi16bits_v2bf16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[0:1], off ; GX900-NEXT: global_load_dword v5, v[2:3], off ; GX900-NEXT: s_mov_b32 s4, 0x7060302 ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: hi16bits_v2bf16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[0:1], off ; GFX942-NEXT: global_load_dword v5, v[2:3], off ; GFX942-NEXT: s_mov_b32 s0, 0x7060302 ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_perm_b32 v0, v5, v4, s0 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: hi16bits_v2bf16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: hi16bits_v2bf16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: hi16bits_v2bf16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4 %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> ret <2 x bfloat> %vy1.2.vec.insert } define <2 x bfloat> @low16hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GX900-LABEL: low16hi16bits_v2bf16: ; GX900: ; %bb.0: ; %entry ; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GX900-NEXT: global_load_dword v4, v[2:3], off ; GX900-NEXT: global_load_dword v5, v[0:1], off ; GX900-NEXT: s_mov_b32 s4, 0xffff ; GX900-NEXT: s_waitcnt vmcnt(0) ; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4 ; GX900-NEXT: s_setpc_b64 s[30:31] ; ; GFX942-LABEL: low16hi16bits_v2bf16: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942-NEXT: global_load_dword v4, v[2:3], off ; GFX942-NEXT: global_load_dword v5, v[0:1], off ; GFX942-NEXT: s_mov_b32 s0, 0xffff ; GFX942-NEXT: s_waitcnt vmcnt(0) ; GFX942-NEXT: v_bfi_b32 v0, s0, v5, v4 ; GFX942-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16hi16bits_v2bf16: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: low16hi16bits_v2bf16: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: low16hi16bits_v2bf16: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4 %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> ret <2 x bfloat> %vy1.2.vec.insert } define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { ; GFX9-LABEL: hi16low16bits: ; GFX9: ; %bb.0: ; %entry ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v4, v[0:1], off ; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: hi16low16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v4, v[0:1], off ; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v5, v4, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: hi16low16bits: ; GFX11-TRUE16: ; %bb.0: ; %entry ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: hi16low16bits: ; GFX11-FAKE16: ; %bb.0: ; %entry ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v1, v0, 16 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4 %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4 %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> ret <2 x bfloat> %vy1.2.vec.insert } define <2 x bfloat> @v2bfloat_hi16bits(ptr addrspace(1) %x0) { ; GFX9-LABEL: v2bfloat_hi16bits: ; GFX9: ; %bb.0: ; %entry ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v2bfloat_hi16bits: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v2bfloat_hi16bits: ; GFX11: ; %bb.0: ; %entry ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %load0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4 %insert1 = insertelement <2 x bfloat> poison, bfloat 0.0, i32 0 %insert2 = insertelement <2 x bfloat> %insert1, bfloat 0.0, i32 1 %vec.ret = shufflevector <2 x bfloat> %insert2, <2 x bfloat> %load0, <2 x i32> ret <2 x bfloat> %vec.ret } define void @shuffle_v8bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v8bf16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[6:7], v[0:1], off ; GFX10-NEXT: global_load_dwordx2 v[8:9], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <8 x i32> store <8 x bfloat> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v16bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v16bf16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16bf16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v16bf16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <8 x bfloat> %val0, <8 x bfloat> %val1, <16 x i32> store <16 x bfloat> %shuffle, ptr addrspace(1) %out ret void } define void @shuffle_v32bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { ; GFX9-LABEL: shuffle_v32bf16_concat: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX9-NEXT: s_waitcnt vmcnt(3) ; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32bf16_concat: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[6:9], v[2:3], off ; GFX10-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: global_load_dwordx4 v[14:17], v[0:1], off ; GFX10-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 ; GFX10-NEXT: s_waitcnt vmcnt(3) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 ; GFX10-NEXT: s_waitcnt vmcnt(2) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[14:17], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v32bf16_concat: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[6:9], v[2:3], off ; GFX11-NEXT: global_load_b128 v[10:13], v[2:3], off offset:16 ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b128 v[14:17], v[0:1], off ; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off offset:16 ; GFX11-NEXT: s_waitcnt vmcnt(3) ; GFX11-NEXT: global_store_b128 v[4:5], v[6:9], off offset:32 ; GFX11-NEXT: s_waitcnt vmcnt(2) ; GFX11-NEXT: global_store_b128 v[4:5], v[10:13], off offset:48 ; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: global_store_b128 v[4:5], v[14:17], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off offset:16 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <16 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <16 x bfloat>, ptr addrspace(1) %arg1 %shuffle = shufflevector <16 x bfloat> %val0, <16 x bfloat> %val1, <32 x i32> store <32 x bfloat> %shuffle, ptr addrspace(1) %out ret void } declare <2 x half> @llvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>) #0 declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>) #0 declare i32 @llvm.amdgcn.workitem.id.x() #0 attributes #0 = { nounwind readnone speculatable }