diff options
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/select-undef.ll | 596 |
1 files changed, 501 insertions, 95 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/select-undef.ll b/llvm/test/CodeGen/AMDGPU/select-undef.ll index 666c38b..ec3781f 100644 --- a/llvm/test/CodeGen/AMDGPU/select-undef.ll +++ b/llvm/test/CodeGen/AMDGPU/select-undef.ll @@ -1,39 +1,46 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -amdgpu-scalar-ir-passes=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN %s -; GCN-LABEL: {{^}}select_undef_lhs: -; GCN: s_waitcnt -; GCN-NOT: v_cmp -; GCN-NOT: v_cndmask -; GCN-NEXT: s_setpc_b64 define float @select_undef_lhs(float %val, i1 %cond) { +; GCN-LABEL: select_undef_lhs: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: s_setpc_b64 s[30:31] %sel = select i1 %cond, float poison, float %val ret float %sel } -; GCN-LABEL: {{^}}select_undef_rhs: -; GCN: s_waitcnt -; GCN-NOT: v_cmp -; GCN-NOT: v_cndmask -; GCN-NEXT: s_setpc_b64 define float @select_undef_rhs(float %val, i1 %cond) { +; GCN-LABEL: select_undef_rhs: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: s_setpc_b64 s[30:31] %sel = select i1 %cond, float %val, float poison ret float %sel } -; GCN-LABEL: {{^}}select_undef_n1: -; GCN: v_mov_b32_e32 [[RES:v[0-9]+]], 1.0 -; GCN: store_dword {{[^,]+}}, [[RES]] define void @select_undef_n1(ptr addrspace(1) %a, i32 %c) { +; GCN-LABEL: select_undef_n1: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v2, 1.0 +; GCN-NEXT: global_store_dword v[0:1], v2, off +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: s_setpc_b64 s[30:31] %cc = icmp eq i32 %c, 0 %sel = select i1 %cc, float 1.000000e+00, float poison store float %sel, ptr addrspace(1) %a ret void } -; GCN-LABEL: {{^}}select_undef_n2: -; GCN: v_mov_b32_e32 [[RES:v[0-9]+]], 1.0 -; GCN: store_dword {{[^,]+}}, [[RES]] define void @select_undef_n2(ptr addrspace(1) %a, i32 %c) { +; GCN-LABEL: select_undef_n2: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_mov_b32_e32 v2, 1.0 +; GCN-NEXT: global_store_dword v[0:1], v2, off +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: s_setpc_b64 s[30:31] %cc = icmp eq i32 %c, 0 %sel = select i1 %cc, float poison, float 1.000000e+00 store float %sel, ptr addrspace(1) %a @@ -44,11 +51,36 @@ declare float @llvm.amdgcn.rcp.f32(float) ; Make sure the vector undef isn't lowered into 0s. -; GCN-LABEL: {{^}}undef_v6f32: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v6f32(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v6f32: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB4_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[6:9], v0 +; GCN-NEXT: ds_read_b64 v[10:11], v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_add_f32_e32 v3, v9, v3 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_f32_e32 v5, v11, v5 +; GCN-NEXT: v_add_f32_e32 v4, v10, v4 +; GCN-NEXT: v_add_f32_e32 v2, v8, v2 +; GCN-NEXT: v_add_f32_e32 v1, v7, v1 +; GCN-NEXT: v_add_f32_e32 v0, v6, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB4_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b64 v0, v[4:5] +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -63,11 +95,36 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v6i32: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v6i32(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v6i32: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB5_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[6:9], v0 +; GCN-NEXT: ds_read_b64 v[10:11], v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_add_u32_e32 v3, v9, v3 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_u32_e32 v5, v11, v5 +; GCN-NEXT: v_add_u32_e32 v4, v10, v4 +; GCN-NEXT: v_add_u32_e32 v2, v8, v2 +; GCN-NEXT: v_add_u32_e32 v1, v7, v1 +; GCN-NEXT: v_add_u32_e32 v0, v6, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB5_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b64 v0, v[4:5] +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -83,11 +140,35 @@ ret: } ; Make sure the vector undef isn't lowered into 0s. -; GCN-LABEL: {{^}}undef_v5f32: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v5f32(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v5f32: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB6_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[5:8], v0 +; GCN-NEXT: ds_read_b32 v9, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_add_f32_e32 v3, v8, v3 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_f32_e32 v4, v9, v4 +; GCN-NEXT: v_add_f32_e32 v2, v7, v2 +; GCN-NEXT: v_add_f32_e32 v1, v6, v1 +; GCN-NEXT: v_add_f32_e32 v0, v5, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB6_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b32 v0, v4 +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -102,11 +183,35 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v5i32: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v5i32(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v5i32: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB7_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[5:8], v0 +; GCN-NEXT: ds_read_b32 v9, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_add_u32_e32 v3, v8, v3 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_u32_e32 v4, v9, v4 +; GCN-NEXT: v_add_u32_e32 v2, v7, v2 +; GCN-NEXT: v_add_u32_e32 v1, v6, v1 +; GCN-NEXT: v_add_u32_e32 v0, v5, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB7_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b32 v0, v4 +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -122,11 +227,35 @@ ret: } ; Make sure the vector undef isn't lowered into 0s. -; GCN-LABEL: {{^}}undef_v3f64: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v3f64(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v3f64: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0 +; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s3, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_mov_b32_e32 v6, s2 +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GCN-NEXT: .LBB8_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[7:10], v6 +; GCN-NEXT: ds_read_b64 v[11:12], v6 offset:16 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_add_f64 v[2:3], v[9:10], v[2:3] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_f64 v[4:5], v[11:12], v[4:5] +; GCN-NEXT: v_add_f64 v[0:1], v[7:8], v[0:1] +; GCN-NEXT: s_cbranch_vccnz .LBB8_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: v_mov_b32_e32 v6, s2 +; GCN-NEXT: ds_write_b64 v6, v[4:5] offset:16 +; GCN-NEXT: ds_write_b128 v6, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -141,11 +270,38 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v3i64: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v3i64(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v3i64: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0 +; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s5, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_mov_b32_e32 v6, s4 +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GCN-NEXT: .LBB9_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[7:10], v6 +; GCN-NEXT: ds_read_b64 v[11:12], v6 offset:16 +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_add_co_u32_e64 v0, s[2:3], v7, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_co_u32_e32 v4, vcc, v11, v4 +; GCN-NEXT: v_addc_co_u32_e32 v5, vcc, v12, v5, vcc +; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v9, v2 +; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v10, v3, vcc +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: v_addc_co_u32_e64 v1, s[2:3], v8, v1, s[2:3] +; GCN-NEXT: s_cbranch_vccnz .LBB9_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: v_mov_b32_e32 v6, s4 +; GCN-NEXT: ds_write_b64 v6, v[4:5] offset:16 +; GCN-NEXT: ds_write_b128 v6, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -161,11 +317,30 @@ ret: } ; Make sure the vector undef isn't lowered into 0s. -; GCN-LABEL: {{^}}undef_v4f16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v4f16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v4f16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s3, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_mov_b32_e32 v2, s2 +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB10_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b64 v[3:4], v2 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_f16 v1, v4, v1 +; GCN-NEXT: v_pk_add_f16 v0, v3, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB10_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: v_mov_b32_e32 v2, s2 +; GCN-NEXT: ds_write_b64 v2, v[0:1] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -180,11 +355,30 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v4i16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v4i16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v4i16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s3, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_mov_b32_e32 v2, s2 +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB11_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b64 v[3:4], v2 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v1, v4, v1 +; GCN-NEXT: v_pk_add_u16 v0, v3, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB11_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: v_mov_b32_e32 v2, s2 +; GCN-NEXT: ds_write_b64 v2, v[0:1] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -200,11 +394,29 @@ ret: } ; Make sure the vector undef isn't lowered into 0s. -; GCN-LABEL: {{^}}undef_v2f16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v2f16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v2f16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s3, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1] +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v1 +; GCN-NEXT: ; implicit-def: $vgpr1 +; GCN-NEXT: .LBB12_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_f16 v1, v2, v1 +; GCN-NEXT: s_cbranch_vccnz .LBB12_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_write_b32 v0, v1 +; GCN-NEXT: s_endpgm entry: br label %loop @@ -219,11 +431,29 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v2i16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v2i16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v2i16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s3, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1] +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v1 +; GCN-NEXT: ; implicit-def: $vgpr1 +; GCN-NEXT: .LBB13_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v1, v2, v1 +; GCN-NEXT: s_cbranch_vccnz .LBB13_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: v_mov_b32_e32 v0, s2 +; GCN-NEXT: ds_write_b32 v0, v1 +; GCN-NEXT: s_endpgm entry: br label %loop @@ -241,14 +471,17 @@ ret: ; We were expanding undef vectors into zero vectors. Optimizations ; would then see we used no elements of the vector, and reform the ; undef vector resulting in a combiner loop. -; GCN-LABEL: {{^}}inf_loop_undef_vector: -; GCN: s_waitcnt -; GCN-NEXT: v_mad_u64_u32 -; GCN-NEXT: v_mul_lo_u32 -; GCN-NEXT: v_mul_lo_u32 -; GCN-NEXT: v_add3_u32 -; GCN-NEXT: global_store_dwordx2 define void @inf_loop_undef_vector(<6 x float> %arg, float %arg1, i64 %arg2) { +; GCN-LABEL: inf_loop_undef_vector: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v7, v[0:1] +; GCN-NEXT: v_mul_lo_u32 v2, v6, v8 +; GCN-NEXT: v_mul_lo_u32 v3, v3, v7 +; GCN-NEXT: v_add3_u32 v1, v3, v1, v2 +; GCN-NEXT: global_store_dwordx2 v[0:1], v[0:1], off +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: s_setpc_b64 s[30:31] %i = insertelement <6 x float> %arg, float %arg1, i64 2 %i3 = bitcast <6 x float> %i to <3 x i64> %i4 = extractelement <3 x i64> %i3, i64 0 @@ -259,11 +492,27 @@ define void @inf_loop_undef_vector(<6 x float> %arg, float %arg1, i64 %arg2) { ret void } -; GCN-LABEL: {{^}}undef_bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0 -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0 -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB15_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_u16 v1, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_add_u32_e32 v0, v1, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB15_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b16 v0, v0 +; GCN-NEXT: s_endpgm entry: br label %loop @@ -281,11 +530,27 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v2bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v2bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v2bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB16_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b32 v1, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v0, v1, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB16_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b32 v0, v0 +; GCN-NEXT: s_endpgm entry: br label %loop @@ -303,11 +568,32 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v3bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v3bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v3bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB17_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b32 v2, v0 +; GCN-NEXT: ds_read_u16 v3, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_pk_add_u16 v0, v2, v0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v1, v3, v1 +; GCN-NEXT: s_cbranch_vccnz .LBB17_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b16 v0, v1 +; GCN-NEXT: ds_write_b32 v0, v0 +; GCN-NEXT: s_endpgm entry: br label %loop @@ -325,11 +611,28 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v4bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v4bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v4bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB18_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b64 v[2:3], v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v1, v3, v1 +; GCN-NEXT: v_pk_add_u16 v0, v2, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB18_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b64 v0, v[0:1] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -347,11 +650,33 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v6bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v6bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v6bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr2 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB19_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b64 v[3:4], v0 +; GCN-NEXT: ds_read_b32 v5, v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(1) +; GCN-NEXT: v_pk_add_u16 v1, v4, v1 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v2, v5, v2 +; GCN-NEXT: v_pk_add_u16 v0, v3, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB19_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b32 v0, v2 +; GCN-NEXT: ds_write_b64 v0, v[0:1] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -369,11 +694,30 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v8bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v8bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v8bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB20_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[4:7], v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v3, v7, v3 +; GCN-NEXT: v_pk_add_u16 v2, v6, v2 +; GCN-NEXT: v_pk_add_u16 v1, v5, v1 +; GCN-NEXT: v_pk_add_u16 v0, v4, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB20_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -391,11 +735,36 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v16bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v16bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v16bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr4 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB21_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[8:11], v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v7, v11, v7 +; GCN-NEXT: v_pk_add_u16 v6, v10, v6 +; GCN-NEXT: v_pk_add_u16 v5, v9, v5 +; GCN-NEXT: v_pk_add_u16 v4, v8, v4 +; GCN-NEXT: v_pk_add_u16 v3, v11, v3 +; GCN-NEXT: v_pk_add_u16 v2, v10, v2 +; GCN-NEXT: v_pk_add_u16 v1, v9, v1 +; GCN-NEXT: v_pk_add_u16 v0, v8, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB21_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b128 v0, v[4:7] +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop @@ -413,11 +782,48 @@ ret: ret void } -; GCN-LABEL: {{^}}undef_v32bf16: -; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} -; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} -; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v32bf16(ptr addrspace(3) %ptr, i1 %cond) { +; GCN-LABEL: undef_v32bf16: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_load_dword s0, s[8:9], 0x4 +; GCN-NEXT: ; implicit-def: $vgpr4 +; GCN-NEXT: ; implicit-def: $vgpr8 +; GCN-NEXT: ; implicit-def: $vgpr12 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] +; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 +; GCN-NEXT: ; implicit-def: $vgpr0 +; GCN-NEXT: .LBB22_1: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ds_read_b128 v[16:19], v0 +; GCN-NEXT: s_and_b64 vcc, exec, s[0:1] +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: v_pk_add_u16 v15, v19, v15 +; GCN-NEXT: v_pk_add_u16 v14, v18, v14 +; GCN-NEXT: v_pk_add_u16 v13, v17, v13 +; GCN-NEXT: v_pk_add_u16 v12, v16, v12 +; GCN-NEXT: v_pk_add_u16 v11, v19, v11 +; GCN-NEXT: v_pk_add_u16 v10, v18, v10 +; GCN-NEXT: v_pk_add_u16 v9, v17, v9 +; GCN-NEXT: v_pk_add_u16 v8, v16, v8 +; GCN-NEXT: v_pk_add_u16 v7, v19, v7 +; GCN-NEXT: v_pk_add_u16 v6, v18, v6 +; GCN-NEXT: v_pk_add_u16 v5, v17, v5 +; GCN-NEXT: v_pk_add_u16 v4, v16, v4 +; GCN-NEXT: v_pk_add_u16 v3, v19, v3 +; GCN-NEXT: v_pk_add_u16 v2, v18, v2 +; GCN-NEXT: v_pk_add_u16 v1, v17, v1 +; GCN-NEXT: v_pk_add_u16 v0, v16, v0 +; GCN-NEXT: s_cbranch_vccnz .LBB22_1 +; GCN-NEXT: ; %bb.2: ; %ret +; GCN-NEXT: ds_write_b128 v0, v[12:15] +; GCN-NEXT: ds_write_b128 v0, v[8:11] +; GCN-NEXT: ds_write_b128 v0, v[4:7] +; GCN-NEXT: ds_write_b128 v0, v[0:3] +; GCN-NEXT: s_endpgm entry: br label %loop |