diff options
Diffstat (limited to 'llvm/test/CodeGen/AMDGPU')
107 files changed, 45773 insertions, 26461 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir index cebdffc..eba64b8 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir @@ -223,37 +223,37 @@ body: | ; GFX7-LABEL: name: load_atomic_flat_v2s32_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s32>)) - ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>)) + ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX9-LABEL: name: load_atomic_flat_v2s32_seq_cst ; GFX9: liveins: $vgpr0_vgpr1 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s32>)) - ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>)) + ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX10-LABEL: name: load_atomic_flat_v2s32_seq_cst ; GFX10: liveins: $vgpr0_vgpr1 ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s32>)) - ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX10-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>)) + ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX11-LABEL: name: load_atomic_flat_v2s32_seq_cst ; GFX11: liveins: $vgpr0_vgpr1 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s32>)) - ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX11-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>)) + ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX12-LABEL: name: load_atomic_flat_v2s32_seq_cst ; GFX12: liveins: $vgpr0_vgpr1 ; GFX12-NEXT: {{ $}} - ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s32>)) - ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX12-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>)) + ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] %0:vgpr(p0) = COPY $vgpr0_vgpr1 %1:vgpr(<2 x s32>) = G_LOAD %0 :: (load seq_cst (<2 x s32>), align 8, addrspace 0) $vgpr0_vgpr1 = COPY %1 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir index eafc96d..474f130 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir @@ -252,30 +252,30 @@ body: | ; GFX7-LABEL: name: load_atomic_global_v2s32_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s32>), addrspace 1) - ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>), addrspace 1) + ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX7-FLAT-LABEL: name: load_atomic_global_v2s32_seq_cst ; GFX7-FLAT: liveins: $vgpr0_vgpr1 ; GFX7-FLAT-NEXT: {{ $}} - ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s32>), addrspace 1) - ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s32>), addrspace 1) + ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX9-LABEL: name: load_atomic_global_v2s32_seq_cst ; GFX9: liveins: $vgpr0_vgpr1 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s32>), addrspace 1) - ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 1) + ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]] ; ; GFX10-LABEL: name: load_atomic_global_v2s32_seq_cst ; GFX10: liveins: $vgpr0_vgpr1 ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s32>), addrspace 1) - ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>) + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s32>), addrspace 1) + ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]] %0:vgpr(p1) = COPY $vgpr0_vgpr1 %1:vgpr(<2 x s32>) = G_LOAD %0 :: (load seq_cst (<2 x s32>), align 8, addrspace 1) $vgpr0_vgpr1 = COPY %1 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir index 2675295..ae010a8 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-atomic-flat.mir @@ -22,6 +22,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2 ; GFX7-NEXT: FLAT_STORE_DWORD [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (s32)) + ; ; GFX9-LABEL: name: atomic_store_flat_s32_seq_cst ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2 ; GFX9-NEXT: {{ $}} @@ -51,6 +52,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2 ; GFX7-NEXT: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p0) :: (store seq_cst (<2 x s16>)) + ; ; GFX9-LABEL: name: atomic_store_flat_v2s16_seq_cst ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2 ; GFX9-NEXT: {{ $}} @@ -80,6 +82,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2 ; GFX7-NEXT: G_STORE [[COPY]](p3), [[COPY1]](p0) :: (store seq_cst (p3)) + ; ; GFX9-LABEL: name: atomic_store_flat_p3_seq_cst ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2 ; GFX9-NEXT: {{ $}} @@ -109,6 +112,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2 ; GFX7-NEXT: G_STORE [[COPY]](p5), [[COPY1]](p0) :: (store seq_cst (p5)) + ; ; GFX9-LABEL: name: atomic_store_flat_p5_seq_cst ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2 ; GFX9-NEXT: {{ $}} @@ -138,6 +142,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p6) = COPY $vgpr0 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr1_vgpr2 ; GFX7-NEXT: G_STORE [[COPY]](p6), [[COPY1]](p0) :: (store seq_cst (p6)) + ; ; GFX9-LABEL: name: atomic_store_flat_p6_seq_cst ; GFX9: liveins: $vgpr0, $vgpr1_vgpr2 ; GFX9-NEXT: {{ $}} @@ -167,6 +172,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3 ; GFX7-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (s64)) + ; ; GFX9-LABEL: name: atomic_store_flat_s64_seq_cst ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3 ; GFX9-NEXT: {{ $}} @@ -193,15 +199,16 @@ body: | ; GFX7-LABEL: name: atomic_store_flat_v2s32_seq_cst ; GFX7: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3 - ; GFX7-NEXT: G_STORE [[COPY]](<2 x s32>), [[COPY1]](p0) :: (store seq_cst (<2 x s32>)) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3 + ; GFX7-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s32>)) + ; ; GFX9-LABEL: name: atomic_store_flat_v2s32_seq_cst ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3 - ; GFX9-NEXT: G_STORE [[COPY]](<2 x s32>), [[COPY1]](p0) :: (store seq_cst (<2 x s32>)) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3 + ; GFX9-NEXT: FLAT_STORE_DWORDX2 [[COPY1]], [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (store seq_cst (<2 x s32>)) %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1 %1:vgpr(p0) = COPY $vgpr2_vgpr3 G_STORE %0, %1 :: (store seq_cst (<2 x s32>), align 8, addrspace 0) @@ -225,6 +232,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3 ; GFX7-NEXT: G_STORE [[COPY]](<4 x s16>), [[COPY1]](p0) :: (store seq_cst (<4 x s16>)) + ; ; GFX9-LABEL: name: atomic_store_flat_v4s16_seq_cst ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3 ; GFX9-NEXT: {{ $}} @@ -254,6 +262,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3 ; GFX7-NEXT: G_STORE [[COPY]](p0), [[COPY1]](p0) :: (store seq_cst (p0)) + ; ; GFX9-LABEL: name: atomic_store_flat_p0_seq_cst ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3 ; GFX9-NEXT: {{ $}} @@ -282,6 +291,7 @@ body: | ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 ; GFX7-NEXT: [[COPY1:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3 ; GFX7-NEXT: G_STORE [[COPY]](p1), [[COPY1]](p0) :: (store seq_cst (p1)) + ; ; GFX9-LABEL: name: atomic_store_flat_p1_seq_cst ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3 ; GFX9-NEXT: {{ $}} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll index 530f4cf..1eb8457 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i32.ll @@ -254,27 +254,13 @@ define i32 @v_srem_i32_pow2k_denom(i32 %num) { ; CHECK-LABEL: v_srem_i32_pow2k_denom: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; CHECK-NEXT: v_rcp_iflag_f32_e32 v2, 0x45800000 -; CHECK-NEXT: v_mov_b32_e32 v3, 0xfffff000 -; CHECK-NEXT: v_mov_b32_e32 v4, 0x1000 -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; CHECK-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1 -; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2 -; CHECK-NEXT: v_mul_lo_u32 v3, v2, v3 -; CHECK-NEXT: v_mul_hi_u32 v3, v2, v3 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; CHECK-NEXT: v_mul_hi_u32 v2, v0, v2 -; CHECK-NEXT: v_lshlrev_b32_e32 v2, 12, v2 -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, 0xfffff000, v0 -; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; CHECK-NEXT: v_add_i32_e32 v2, vcc, 0xfffff000, v0 -; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1 +; CHECK-NEXT: v_mov_b32_e32 v1, 0x80000001 +; CHECK-NEXT: v_mul_hi_i32 v1, v0, v1 +; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v0 +; CHECK-NEXT: v_ashrrev_i32_e32 v1, 11, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, 31, v1 +; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; CHECK-NEXT: v_lshlrev_b32_e32 v1, 12, v1 ; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; CHECK-NEXT: s_setpc_b64 s[30:31] %result = srem i32 %num, 4096 @@ -327,42 +313,21 @@ define <2 x i32> @v_srem_v2i32_pow2k_denom(<2 x i32> %num) { ; CGP-LABEL: v_srem_v2i32_pow2k_denom: ; CGP: ; %bb.0: ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CGP-NEXT: v_ashrrev_i32_e32 v2, 31, v0 -; CGP-NEXT: v_rcp_iflag_f32_e32 v3, 0x45800000 -; CGP-NEXT: v_mov_b32_e32 v4, 0xfffff000 -; CGP-NEXT: v_mov_b32_e32 v5, 0x1000 -; CGP-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v6 -; CGP-NEXT: v_xor_b32_e32 v0, v0, v2 -; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3 -; CGP-NEXT: v_xor_b32_e32 v1, v1, v6 -; CGP-NEXT: v_mul_lo_u32 v7, v3, v4 -; CGP-NEXT: v_mul_hi_u32 v7, v3, v7 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; CGP-NEXT: v_mul_hi_u32 v7, v0, v3 -; CGP-NEXT: v_mul_hi_u32 v3, v1, v3 -; CGP-NEXT: v_lshlrev_b32_e32 v7, 12, v7 +; CGP-NEXT: v_mov_b32_e32 v2, 0x80000001 +; CGP-NEXT: v_mul_hi_i32 v3, v0, v2 +; CGP-NEXT: v_mul_hi_i32 v2, v1, v2 +; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v0 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v1 +; CGP-NEXT: v_ashrrev_i32_e32 v3, 11, v3 +; CGP-NEXT: v_ashrrev_i32_e32 v2, 11, v2 +; CGP-NEXT: v_lshrrev_b32_e32 v4, 31, v3 +; CGP-NEXT: v_lshrrev_b32_e32 v5, 31, v2 +; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v4 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v5 ; CGP-NEXT: v_lshlrev_b32_e32 v3, 12, v3 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v0, v4 -; CGP-NEXT: v_add_i32_e32 v7, vcc, 0xfffff000, v1 -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc -; CGP-NEXT: v_add_i32_e32 v3, vcc, v0, v4 -; CGP-NEXT: v_add_i32_e32 v4, vcc, 0xfffff000, v1 -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; CGP-NEXT: v_xor_b32_e32 v0, v0, v2 -; CGP-NEXT: v_xor_b32_e32 v1, v1, v6 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v6 +; CGP-NEXT: v_lshlrev_b32_e32 v2, 12, v2 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v3 +; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; CGP-NEXT: s_setpc_b64 s[30:31] %result = srem <2 x i32> %num, <i32 4096, i32 4096> ret <2 x i32> %result @@ -372,27 +337,14 @@ define i32 @v_srem_i32_oddk_denom(i32 %num) { ; CHECK-LABEL: v_srem_i32_oddk_denom: ; CHECK: ; %bb.0: ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v0 -; CHECK-NEXT: v_rcp_iflag_f32_e32 v2, 0x4996c7d8 -; CHECK-NEXT: v_mov_b32_e32 v3, 0xffed2705 -; CHECK-NEXT: v_mov_b32_e32 v4, 0x12d8fb -; CHECK-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; CHECK-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1 -; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2 -; CHECK-NEXT: v_mul_lo_u32 v3, v2, v3 -; CHECK-NEXT: v_mul_hi_u32 v3, v2, v3 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; CHECK-NEXT: v_mul_hi_u32 v2, v0, v2 -; CHECK-NEXT: v_mul_lo_u32 v2, v2, v4 -; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; CHECK-NEXT: v_add_i32_e32 v2, vcc, 0xffed2705, v0 -; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; CHECK-NEXT: v_add_i32_e32 v2, vcc, 0xffed2705, v0 -; CHECK-NEXT: v_cmp_ge_u32_e32 vcc, v0, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; CHECK-NEXT: v_xor_b32_e32 v0, v0, v1 +; CHECK-NEXT: v_mov_b32_e32 v1, 0xd9528441 +; CHECK-NEXT: v_mul_hi_i32 v1, v0, v1 +; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v0 +; CHECK-NEXT: v_ashrrev_i32_e32 v1, 20, v1 +; CHECK-NEXT: v_lshrrev_b32_e32 v2, 31, v1 +; CHECK-NEXT: v_add_i32_e32 v1, vcc, v1, v2 +; CHECK-NEXT: v_mov_b32_e32 v2, 0x12d8fb +; CHECK-NEXT: v_mul_lo_u32 v1, v1, v2 ; CHECK-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 ; CHECK-NEXT: s_setpc_b64 s[30:31] %result = srem i32 %num, 1235195 @@ -445,42 +397,22 @@ define <2 x i32> @v_srem_v2i32_oddk_denom(<2 x i32> %num) { ; CGP-LABEL: v_srem_v2i32_oddk_denom: ; CGP: ; %bb.0: ; CGP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CGP-NEXT: v_ashrrev_i32_e32 v2, 31, v0 -; CGP-NEXT: v_rcp_iflag_f32_e32 v3, 0x4996c7d8 -; CGP-NEXT: v_mov_b32_e32 v4, 0xffed2705 -; CGP-NEXT: v_mov_b32_e32 v5, 0x12d8fb -; CGP-NEXT: v_ashrrev_i32_e32 v6, 31, v1 -; CGP-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_mul_f32_e32 v3, 0x4f7ffffe, v3 -; CGP-NEXT: v_add_i32_e32 v1, vcc, v1, v6 -; CGP-NEXT: v_xor_b32_e32 v0, v0, v2 -; CGP-NEXT: v_cvt_u32_f32_e32 v3, v3 -; CGP-NEXT: v_xor_b32_e32 v1, v1, v6 -; CGP-NEXT: v_mul_lo_u32 v7, v3, v4 -; CGP-NEXT: v_mul_hi_u32 v7, v3, v7 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v3, v7 -; CGP-NEXT: v_mul_hi_u32 v7, v0, v3 -; CGP-NEXT: v_mul_hi_u32 v3, v1, v3 -; CGP-NEXT: v_mul_lo_u32 v7, v7, v5 -; CGP-NEXT: v_mul_lo_u32 v3, v3, v5 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v7 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v3 -; CGP-NEXT: v_add_i32_e32 v3, vcc, v0, v4 -; CGP-NEXT: v_add_i32_e32 v7, vcc, 0xffed2705, v1 -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc -; CGP-NEXT: v_add_i32_e32 v3, vcc, v0, v4 -; CGP-NEXT: v_add_i32_e32 v4, vcc, 0xffed2705, v1 -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v0, v5 -; CGP-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; CGP-NEXT: v_cmp_ge_u32_e32 vcc, v1, v5 -; CGP-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; CGP-NEXT: v_xor_b32_e32 v0, v0, v2 -; CGP-NEXT: v_xor_b32_e32 v1, v1, v6 -; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v2 -; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v6 +; CGP-NEXT: v_mov_b32_e32 v2, 0xd9528441 +; CGP-NEXT: v_mov_b32_e32 v3, 0x12d8fb +; CGP-NEXT: v_mul_hi_i32 v4, v0, v2 +; CGP-NEXT: v_mul_hi_i32 v2, v1, v2 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v0 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v1 +; CGP-NEXT: v_ashrrev_i32_e32 v4, 20, v4 +; CGP-NEXT: v_ashrrev_i32_e32 v2, 20, v2 +; CGP-NEXT: v_lshrrev_b32_e32 v5, 31, v4 +; CGP-NEXT: v_lshrrev_b32_e32 v6, 31, v2 +; CGP-NEXT: v_add_i32_e32 v4, vcc, v4, v5 +; CGP-NEXT: v_add_i32_e32 v2, vcc, v2, v6 +; CGP-NEXT: v_mul_lo_u32 v4, v4, v3 +; CGP-NEXT: v_mul_lo_u32 v2, v2, v3 +; CGP-NEXT: v_sub_i32_e32 v0, vcc, v0, v4 +; CGP-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; CGP-NEXT: s_setpc_b64 s[30:31] %result = srem <2 x i32> %num, <i32 1235195, i32 1235195> ret <2 x i32> %result diff --git a/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir b/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir index 2bd1b8b..d22a4b9 100644 --- a/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/accvgpr-copy.mir @@ -45,6 +45,9 @@ define amdgpu_kernel void @copy_agpr_to_agpr_tuple() #0 { ret void } define amdgpu_kernel void @copy_agpr_to_agpr_tuple_kill() #0 { ret void } + define amdgpu_kernel void @look_for_vgpr_killed() #0 { ret void } + define amdgpu_kernel void @look_for_vgpr_killed_tuple() #0 { ret void } + attributes #0 = { "amdgpu-flat-work-group-size"="1,256" } ... @@ -1517,3 +1520,83 @@ body: | renamable $agpr4_agpr5_agpr6_agpr7 = COPY renamable killed $agpr0_agpr1_agpr2_agpr3, implicit $exec S_ENDPGM 0, implicit $agpr4_agpr5_agpr6_agpr7 ... + +# Make sure the expansion of the a-to-a copy doesn't introduce a use +# after kill of the source vgpr +--- +name: look_for_vgpr_killed +tracksRegLiveness: true +body: | + bb.0: + liveins: $agpr0 + + ; GFX908-LABEL: name: look_for_vgpr_killed + ; GFX908: liveins: $agpr0 + ; GFX908-NEXT: {{ $}} + ; GFX908-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; GFX908-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; GFX908-NEXT: S_NOP 0, implicit $vgpr0 + ; GFX908-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; + ; GFX90A-LABEL: name: look_for_vgpr_killed + ; GFX90A: liveins: $agpr0 + ; GFX90A-NEXT: {{ $}} + ; GFX90A-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; GFX90A-NEXT: S_NOP 0, implicit killed $vgpr0 + ; GFX90A-NEXT: $agpr1 = V_ACCVGPR_MOV_B32 $agpr0, implicit $exec + ; + ; GFX942-LABEL: name: look_for_vgpr_killed + ; GFX942: liveins: $agpr0 + ; GFX942-NEXT: {{ $}} + ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; GFX942-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; GFX942-NEXT: S_NOP 0, implicit killed $vgpr0 + ; GFX942-NEXT: $agpr1 = V_ACCVGPR_MOV_B32 $agpr0, implicit $exec + $vgpr0 = V_MOV_B32_e32 0, implicit $exec + $agpr0 = COPY $vgpr0 + S_NOP 0, implicit killed $vgpr0 + $agpr1 = COPY $agpr0 + +... + +--- +name: look_for_vgpr_killed_tuple +tracksRegLiveness: true +body: | + bb.0: + liveins: $agpr0 + + ; GFX908-LABEL: name: look_for_vgpr_killed_tuple + ; GFX908: liveins: $agpr0 + ; GFX908-NEXT: {{ $}} + ; GFX908-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1 + ; GFX908-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec + ; GFX908-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; GFX908-NEXT: S_NOP 0, implicit $vgpr0_vgpr1 + ; GFX908-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; + ; GFX90A-LABEL: name: look_for_vgpr_killed_tuple + ; GFX90A: liveins: $agpr0 + ; GFX90A-NEXT: {{ $}} + ; GFX90A-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1 + ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec + ; GFX90A-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; GFX90A-NEXT: S_NOP 0, implicit killed $vgpr0_vgpr1 + ; GFX90A-NEXT: $agpr1 = V_ACCVGPR_MOV_B32 $agpr0, implicit $exec + ; + ; GFX942-LABEL: name: look_for_vgpr_killed_tuple + ; GFX942: liveins: $agpr0 + ; GFX942-NEXT: {{ $}} + ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1 + ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec + ; GFX942-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; GFX942-NEXT: S_NOP 0, implicit killed $vgpr0_vgpr1 + ; GFX942-NEXT: $agpr1 = V_ACCVGPR_MOV_B32 $agpr0, implicit $exec + $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1 + $vgpr1 = V_MOV_B32_e32 1, implicit $exec + $agpr0 = COPY $vgpr0 + S_NOP 0, implicit killed $vgpr0_vgpr1 + $agpr1 = COPY $agpr0 + +... diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll index 50d20e9..6cb236d 100644 --- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll +++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll @@ -780,7 +780,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(ptr addrspace(1) %out, ; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 ; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3] glc dlc @@ -789,11 +790,12 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(ptr addrspace(1) %out, ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v1, v0 ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, 0, 16, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v2, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] ; GFX11-TRUE16-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll index 6742ae6..f6465de 100644 --- a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll +++ b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll @@ -6,17 +6,17 @@ define amdgpu_kernel void @remat_constant_voids_spill(ptr addrspace(1) %p) #1 { ; GFX908-LABEL: remat_constant_voids_spill: ; GFX908: ; %bb.0: -; GFX908-NEXT: v_accvgpr_write_b32 a1, 1 -; GFX908-NEXT: v_accvgpr_write_b32 a5, 6 -; GFX908-NEXT: v_accvgpr_write_b32 a6, 7 -; GFX908-NEXT: v_accvgpr_write_b32 a7, 8 -; GFX908-NEXT: v_accvgpr_write_b32 a0, 9 -; GFX908-NEXT: v_accvgpr_write_b32 a2, 2 -; GFX908-NEXT: v_accvgpr_write_b32 a3, 3 -; GFX908-NEXT: v_accvgpr_write_b32 a4, 4 +; GFX908-NEXT: v_accvgpr_write_b32 a0, 1 +; GFX908-NEXT: v_accvgpr_write_b32 a1, 2 +; GFX908-NEXT: v_accvgpr_write_b32 a2, 3 +; GFX908-NEXT: v_accvgpr_write_b32 a3, 4 ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_write_b32 a1, 5 +; GFX908-NEXT: v_accvgpr_write_b32 a0, 5 +; GFX908-NEXT: v_accvgpr_write_b32 a1, 6 +; GFX908-NEXT: v_accvgpr_write_b32 a2, 7 +; GFX908-NEXT: v_accvgpr_write_b32 a3, 8 +; GFX908-NEXT: v_accvgpr_write_b32 a4, 9 ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ;;#ASMEND ; GFX908-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll index cb2f0f2..0d5f538 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll @@ -6309,64 +6309,64 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) @@ -6394,50 +6394,50 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[65:66], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[54:55], 24, v[7:8] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] @@ -6498,50 +6498,50 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v1 ; GFX11-TRUE16-NEXT: .LBB12_4: ; %end @@ -6549,319 +6549,314 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v162.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v1.h, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v161.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v1.h, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v2.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff, v66 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v55, v39 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v67 -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v65 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v2.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v66, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v149.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v39, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v65 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v66 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v148.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v5.l, v33.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v147.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v39, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v33.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v64 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v55, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v144.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v39, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v54, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v54, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v53 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v53, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v9.l, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v132.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v52, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v129.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v128.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v52, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v51, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v51, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v113.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v49, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v112.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v18.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v49, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v102.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v19.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v48, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v20.h, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v48, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v21.l, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v98.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v22.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v23.h, v24.l ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v12.h, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v117.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v16.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v114.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v113.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v112.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v103.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v101.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v96.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v85.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v82.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v80.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v71.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v69.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v65 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v53, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v39, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v38, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v35, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v37, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v25.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v87.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v86.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v24.l, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v37, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v85.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v25.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v84.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v36, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v83.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v26.h, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v36, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v27.l, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v34.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v28.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v71.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v29.h, v30.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v31.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v69.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v30.l, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v31.l, v31.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v32.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v68.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v32.l, v32.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v33, v39 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v32i32_to_v128i8: @@ -15418,63 +15413,63 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:380 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:376 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:372 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:368 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:364 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:372 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:368 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:364 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:360 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:356 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:352 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:348 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:356 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:352 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:348 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:344 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:340 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:336 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:336 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:332 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:328 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:324 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:324 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:320 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:316 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:312 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:316 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:312 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:308 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:304 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:300 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:296 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:292 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:288 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:284 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:292 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:288 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:284 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:280 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:276 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:272 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:268 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:264 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:276 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:272 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:268 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:260 ; GFX11-TRUE16-NEXT: s_clause 0x1f -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:256 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:256 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:252 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:248 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:244 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:240 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:236 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:232 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:228 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:224 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:220 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:216 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:232 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:228 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:224 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:220 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:216 ; GFX11-TRUE16-NEXT: scratch_load_b32 v103, off, s32 offset:388 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:8 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:16 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:24 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:32 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:40 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:8 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:16 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:24 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:40 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v86, off, s32 offset:48 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:56 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:64 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:72 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:80 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:88 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:112 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:64 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:72 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:80 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:88 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:96 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:104 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:112 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:120 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v160, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:136 @@ -15488,146 +15483,144 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v164, off, s32 offset:192 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:200 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v165, off, s32 offset:208 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:212 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:204 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:212 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:204 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:196 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:188 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:180 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:172 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:164 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:156 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 offset:148 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:140 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:124 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:116 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:108 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:100 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:180 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:172 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:156 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:148 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:140 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:124 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v97, off, s32 offset:84 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:76 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v102, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v113, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v114, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v112, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:12 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v115, off, s32 offset:4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v30.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.h, v26.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.h, v24.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v6.l +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v116, off, s32 offset:4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v26.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.h, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v150.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v151.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v148.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.l, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v23.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.l, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v50.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v50.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v49.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v54.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v53.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v64.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(61) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(54) ; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v103 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(53) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v80.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(52) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v81.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.h, 8, v82.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v82.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(50) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v82.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v83.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(49) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v83.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v84.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(48) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.h, 8, v86.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v119.h, 8, v85.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.l, 8, v86.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v86.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(46) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.h, 8, v87.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(45) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v87.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v114.h, 8, v96.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(44) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v96.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v115.h, 8, v97.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v98.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(42) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.l, 8, v99.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.h, 8, v99.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(41) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v99.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(40) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v100.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v101.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v101.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(38) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v160.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v160.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(37) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v160.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v160.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(36) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v161.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v161.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v161.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v161.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(34) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v162.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v162.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v162.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(32) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v163.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v163.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v163.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.l, 8, v163.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v164.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v164.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.l, 8, v164.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v164.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(28) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v165.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v165.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.l, 8, v165.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.l, 8, v69.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.l, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v55.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v55.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v54.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v54.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v52.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v49.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -15641,720 +15634,746 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB14_3: ; %cmp.false -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v149.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v149.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v151.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v148.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v148.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v150.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v150.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v151.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v151.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v145.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v144.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v145.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v144.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v146.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v132.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v135.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v146.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v147.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v131.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v132.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v147.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v119.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v133.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v119.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v130.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v133.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v134.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v118.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v134.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v114.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v128.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v150.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v146.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v151.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v150.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v147.h +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v0.h, v149.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v149.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v144.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v133.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v148.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v145.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v1.h, v147.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v129.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v130.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v113.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v116.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v129.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v102.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v101.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v116.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v117.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v117.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v118.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v98.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v97.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v134.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v2.l, v146.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v131.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v135.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v3.l, v145.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v128.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v118.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v116.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v4.l, v135.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v129.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v134.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v5.l, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v132.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v6.l, v132.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v130.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v112.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v7.l, v130.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v129.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v102.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v97.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v96.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v103.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v112.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v85.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v112.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v113.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v85.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v84.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v83.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v99.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v100.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v80.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v100.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v101.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v80.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v71.h -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v70.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v8.l, v128.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v112.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v119.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v100.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v9.l, v118.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v102.h +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v117.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v97.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v10.l, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v100.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v115.h +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v87.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v11.l, v114.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v11.h, v113.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v85.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v12.l, v113.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v103.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v83.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v71.l -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v86.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v96.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v67.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v87.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v67.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v66.h -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v81.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v82.l -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v82.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v83.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v52.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v48.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v69.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v13.l, v103.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v101.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v81.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v14.l, v101.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v84.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v99.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v71.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v15.l, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v81.h +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v15.h, v98.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v69.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v16.l, v97.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v71.h +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v16.h, v96.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v67.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v68.h -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v69.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v70.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v39.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v55.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v64.h -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v65.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v17.l, v87.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v69.h +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v17.h, v86.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v64.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v18.l, v85.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v67.h +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v18.h, v84.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v19.l, v83.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v65.h +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v19.h, v82.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v20.l, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v20.h, v80.h +; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v21.l, v80.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v21.h, v70.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v53.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v50.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v22.l, v70.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v22.h, v68.h +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v23.l, v68.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v23.h, v66.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v24.l, v66.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v24.h, v65.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v27, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v25.l, v64.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v25.h, v55.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v28, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v26.l, v55.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v26.h, v54.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v27.l, v54.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v28.h, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v28.l, v53.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v28.h, v52.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v33.l, v29.h, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v29.l, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v30.l, v50.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v32.l, v49.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2 ; GFX11-TRUE16-NEXT: .LBB14_4: ; %cmp.true -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v149.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v149.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v148.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v148.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v146.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v147.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v150.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v150.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v151.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v145.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v151.l, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v144.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v151.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v150.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v149.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v149.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v147.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v134.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v133.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v144.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v135.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v135.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v148.l, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v146.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v147.h, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v147.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v132.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v131.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v131.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v119.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v132.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v133.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v6 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v130.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v145.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v131.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v145.h, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v131.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v119.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v118.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v133.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v134.h, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v134.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v128.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v128.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v128.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v135.h, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v135.l, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v129.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.l, v32.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v134.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v118.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v133.l, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v119.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v132.h, v6.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v114.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v114.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v115.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v113.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v116.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v132.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v117.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v130.h, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v114.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v130.l, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v129.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v130.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v116.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v116.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v102.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v101.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v102.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.l, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v112.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v128.l, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v112.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v98.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v117.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v119.h, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v102.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l ; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v118.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v102.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v117.h, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v116.l, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v100.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v115.h, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v97.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v114.h, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v103.l, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v117.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v97.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v103.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v97.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v96.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v85.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v112.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v85.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v112.h, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v113.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v99.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v84.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v99.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v84.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v83.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v80.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v113.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v87.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v113.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v96.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v103.h, v13.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v85.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v103.l, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v86.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v13.h ; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v100.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v80.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v100.h, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v101.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v83.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l ; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v101.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v84.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v99.h, v15.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v81.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v99.l, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, v81.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v15.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v98.l, v16.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v71.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v97.l, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v71.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v86.l, v17.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v71.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v70.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v86.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v71.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v67.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v87.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v96.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v67.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v16.h ; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v87.h, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v81.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v66.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v66.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v52.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v81.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v96.h, v17.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v69.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v87.l, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v69.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v86.h, v18.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v67.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v85.l, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v67.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v19.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v18.h ; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v52.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v82.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v82.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v49.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v48.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v84.h, v19.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v64.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.h, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v65.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v19.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v82.h, v20.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v51.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v82.l, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, v52.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v21.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v80.h, v21.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v80.l, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v68.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v69.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v68.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v69.h, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v39.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v70.l, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v38.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v37.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v21.h ; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v70.h, v22.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v70.l, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v55.h, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v64.h, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v64.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v65.l, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v22.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v68.h, v23.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v68.l, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v23.h ; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v65.h, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v36.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v66.l, v24.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v66.h, v24.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v64.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v26, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v65.l, v25.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v25.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v26.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v27, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v55.l, v26.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v55.h, v26.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v26.h ; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v54.h, v27.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v54.l, v27.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v53.l, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v54.l, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v55.l, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v53.h, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v54.h, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v34.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v33.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v53.h, v28.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v33.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v49.h, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v50.l, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v50.h, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v51.l, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v51.h, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v53.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v52.h, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v50.h, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v51.l, v30.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v50.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.l, 0x300, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v30.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v49.h, v30.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v32.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -42137,64 +42156,64 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) @@ -42222,50 +42241,50 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[65:66], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[54:55], 24, v[7:8] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] @@ -42309,50 +42328,50 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v1 ; GFX11-TRUE16-NEXT: .LBB36_4: ; %end @@ -42360,319 +42379,314 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v162.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v1.h, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v161.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v1.h, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v2.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff, v66 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v55, v39 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v67 -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v65 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v2.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v66, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v149.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v39, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v65 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v66 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v148.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v5.l, v33.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v147.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v39, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v33.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v64 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v55, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v144.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v39, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v54, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v54, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v53 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v53, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v9.l, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v132.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v52, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v129.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v128.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v52, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v51, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v51, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v113.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v49, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v112.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v18.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v49, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v102.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v19.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v48, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v20.h, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v48, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v21.l, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v98.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v22.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v23.h, v24.l ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v12.h, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v117.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v16.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v114.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v113.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v112.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v103.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v101.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v96.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v85.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v82.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v80.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v71.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v69.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v65 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v53, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v39, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v38, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v35, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v37, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v25.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v87.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v86.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v24.l, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v37, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v85.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v25.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v84.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v36, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v83.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v26.h, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v36, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v27.l, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v34.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v28.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v71.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v29.h, v30.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v31.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v69.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v30.l, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v31.l, v31.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v32.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v68.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v32.l, v32.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v33, v39 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v32f32_to_v128i8: @@ -52196,63 +52210,63 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:380 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:376 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:372 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:368 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:364 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:372 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:368 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:364 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:360 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:356 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:352 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:348 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:356 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:352 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:348 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:344 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:340 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:336 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:336 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:332 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:328 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:324 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:324 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:320 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:316 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:312 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:316 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:312 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:308 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:304 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:300 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:296 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:292 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:288 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:284 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:292 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:288 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:284 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:280 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:276 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:272 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:268 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:264 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:276 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:272 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:268 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:260 ; GFX11-TRUE16-NEXT: s_clause 0x1f -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:256 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:256 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:252 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:248 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:244 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:240 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:236 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:232 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:228 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:224 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:220 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:216 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:232 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:228 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:224 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:220 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:216 ; GFX11-TRUE16-NEXT: scratch_load_b32 v103, off, s32 offset:388 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:8 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:16 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:24 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:32 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:40 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:8 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:16 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:24 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:40 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v86, off, s32 offset:48 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:56 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:64 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:72 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:80 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:88 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:112 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:64 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:72 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:80 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:88 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:96 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:104 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:112 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:120 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v160, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:136 @@ -52266,146 +52280,144 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v164, off, s32 offset:192 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:200 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v165, off, s32 offset:208 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:212 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:204 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:212 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:204 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:196 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:188 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:180 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:172 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:164 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:156 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 offset:148 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:140 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:124 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:116 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:108 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:100 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:180 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:172 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:156 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:148 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:140 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:124 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v97, off, s32 offset:84 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:76 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v102, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v113, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v114, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v112, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:12 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v115, off, s32 offset:4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v30.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.h, v26.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.h, v24.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v6.l +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v116, off, s32 offset:4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v26.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.h, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v150.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v151.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v148.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.l, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v23.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.l, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v50.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v50.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v49.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v54.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v53.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v64.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(61) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(54) ; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v103 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(53) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v80.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(52) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v81.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.h, 8, v82.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v82.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(50) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v82.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v83.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(49) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v83.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v84.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(48) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.h, 8, v86.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v119.h, 8, v85.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.l, 8, v86.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v86.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(46) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.h, 8, v87.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(45) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v87.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v114.h, 8, v96.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(44) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v96.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v115.h, 8, v97.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v98.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(42) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.l, 8, v99.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.h, 8, v99.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(41) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v99.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(40) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v100.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v101.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v101.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(38) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v160.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v160.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(37) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v160.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v160.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(36) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v161.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v161.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v161.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v161.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(34) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v162.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v162.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v162.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(32) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v163.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v163.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v163.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.l, 8, v163.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v164.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v164.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.l, 8, v164.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v164.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(28) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v165.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v165.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.l, 8, v165.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.l, 8, v69.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.l, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v55.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v55.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v54.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v54.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v52.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v49.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -52419,720 +52431,746 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB38_3: ; %cmp.false -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v149.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v149.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v151.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v148.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v148.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v150.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v150.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v151.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v151.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v145.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v144.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v145.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v144.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v146.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v132.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v135.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v146.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v147.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v131.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v132.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v147.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v119.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v133.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v119.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v130.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v133.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v134.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v118.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v134.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v114.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v128.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v150.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v146.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v151.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v150.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v147.h +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v0.h, v149.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v149.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v144.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v133.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v148.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v145.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v1.h, v147.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v129.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v130.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v113.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v116.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v129.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v102.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v101.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v116.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v117.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v117.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v118.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v98.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v97.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v134.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v2.l, v146.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v131.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v135.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v3.l, v145.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v128.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v118.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v116.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v4.l, v135.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v129.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v134.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v5.l, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v132.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v6.l, v132.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v130.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v112.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v7.l, v130.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v129.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v102.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v97.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v96.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v103.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v112.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v85.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v112.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v113.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v85.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v84.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v83.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v99.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v100.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v80.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v100.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v101.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v80.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v71.h -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v70.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v8.l, v128.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v112.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v119.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v100.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v9.l, v118.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v102.h +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v117.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v97.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v10.l, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v100.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v115.h +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v87.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v11.l, v114.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v11.h, v113.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v85.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v12.l, v113.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v103.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v83.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v71.l -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v86.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v96.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v67.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v87.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v67.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v66.h -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v81.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v82.l -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v82.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v83.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v52.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v48.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v69.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v13.l, v103.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v101.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v81.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v14.l, v101.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v84.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v99.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v71.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v15.l, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v81.h +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v15.h, v98.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v69.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v16.l, v97.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v71.h +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v16.h, v96.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v67.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v68.h -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v69.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v70.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v39.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v55.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v64.h -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v65.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v17.l, v87.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v69.h +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v17.h, v86.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v64.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v18.l, v85.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v67.h +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v18.h, v84.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v19.l, v83.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v65.h +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v19.h, v82.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v20.l, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v20.h, v80.h +; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v21.l, v80.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v21.h, v70.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v53.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v50.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v22.l, v70.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v22.h, v68.h +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v23.l, v68.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v23.h, v66.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v24.l, v66.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v24.h, v65.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v27, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v25.l, v64.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v25.h, v55.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v28, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v26.l, v55.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v26.h, v54.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v27.l, v54.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v28.h, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v28.l, v53.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v28.h, v52.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v33.l, v29.h, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v29.l, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v30.l, v50.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v32.l, v49.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_2 ; GFX11-TRUE16-NEXT: .LBB38_4: ; %cmp.true -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v149.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v149.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v148.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v148.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v146.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v147.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v150.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v150.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v151.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v145.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v151.l, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v144.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v151.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v150.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v149.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v149.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v147.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v134.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v133.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v144.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v135.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v135.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v148.l, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v146.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v147.h, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v147.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v132.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v131.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v131.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v119.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v132.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v133.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v6 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v130.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v145.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v131.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v145.h, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v131.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v119.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v118.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v133.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v134.h, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v134.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v128.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v128.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v128.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v135.h, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v135.l, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v129.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.l, v32.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v134.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v118.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v133.l, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v119.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v132.h, v6.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v114.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v114.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v115.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v113.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v116.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v132.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v117.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v130.h, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v114.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v130.l, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v129.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v130.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v116.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v116.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v102.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v101.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v102.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.l, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v112.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v128.l, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v112.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v98.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v117.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v119.h, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v102.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l ; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v118.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v102.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v117.h, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v116.l, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v100.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v115.h, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v97.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v114.h, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v103.l, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v117.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v97.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v103.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v97.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v96.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v85.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v112.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v85.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v112.h, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v113.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v99.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v84.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v99.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v84.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v83.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v80.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v113.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v87.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v113.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v96.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v103.h, v13.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v85.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v103.l, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v86.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v13.h ; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v100.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v80.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v100.h, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v101.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v83.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l ; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v101.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v84.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v99.h, v15.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v81.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v99.l, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, v81.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v15.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v98.l, v16.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v71.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v97.l, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v71.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v86.l, v17.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v71.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v70.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v86.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v71.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v67.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v87.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v96.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v67.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v16.h ; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v87.h, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v81.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v66.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v66.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v52.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v81.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v96.h, v17.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v69.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v87.l, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v69.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v86.h, v18.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v67.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v85.l, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v67.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v19.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v18.h ; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v52.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v82.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v82.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v49.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v48.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v84.h, v19.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v64.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.h, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v65.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v19.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v82.h, v20.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v51.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v82.l, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, v52.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v21.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v80.h, v21.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v80.l, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v68.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v69.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v68.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v69.h, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v39.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v70.l, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v38.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v37.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v21.h ; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v70.h, v22.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v70.l, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v55.h, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v64.h, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v64.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v65.l, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v22.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v68.h, v23.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v68.l, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v23.h ; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v65.h, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v36.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v66.l, v24.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v66.h, v24.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v64.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v26, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v65.l, v25.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v25.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v26.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v27, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v55.l, v26.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v55.h, v26.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v26.h ; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v54.h, v27.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v54.l, v27.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v53.l, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v54.l, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v55.l, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v53.h, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v54.h, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v34.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v33.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v53.h, v28.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v33.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v49.h, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v50.l, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v50.h, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v51.l, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v51.h, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v53.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v52.h, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v50.h, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v51.l, v30.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v50.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.l, 0x300, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v30.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v49.h, v30.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v32.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -77900,64 +77938,64 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) @@ -77985,50 +78023,50 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[65:66], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[54:55], 24, v[7:8] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] @@ -78097,50 +78135,50 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v1 ; GFX11-TRUE16-NEXT: .LBB56_4: ; %end @@ -78148,319 +78186,314 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v162.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v1.h, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v161.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v1.h, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v2.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff, v66 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v55, v39 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v67 -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v65 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v2.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v66, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v149.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v39, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v65 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v66 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v148.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v5.l, v33.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v147.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v39, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v33.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v64 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v55, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v144.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v39, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v54, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v54, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v53 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v53, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v9.l, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v132.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v52, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v129.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v128.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v52, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v51, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v51, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v113.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v49, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v112.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v18.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v49, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v102.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v19.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v48, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v20.h, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v48, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v21.l, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v98.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v22.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v23.h, v24.l ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v12.h, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v117.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v16.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v114.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v113.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v112.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v103.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v101.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v96.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v85.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v82.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v80.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v71.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v69.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v65 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v53, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v39, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v38, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v35, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v37, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v25.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v87.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v86.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v24.l, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v37, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v85.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v25.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v84.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v36, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v83.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v26.h, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v36, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v27.l, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v34.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v28.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v71.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v29.h, v30.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v31.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v69.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v30.l, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v31.l, v31.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v32.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v68.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v32.l, v32.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v33, v39 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v16i64_to_v128i8: @@ -87027,63 +87060,63 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:380 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:376 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:372 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:368 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:364 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:372 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:368 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:364 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:360 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:356 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:352 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:348 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:356 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:352 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:348 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:344 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:340 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:336 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:336 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:332 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:328 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:324 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:324 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:320 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:316 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:312 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:316 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:312 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:308 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:304 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:300 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:296 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:292 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:288 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:284 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:292 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:288 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:284 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:280 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:276 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:272 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:268 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:264 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:276 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:272 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:268 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:260 ; GFX11-TRUE16-NEXT: s_clause 0x1f -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:256 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:256 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:252 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:248 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:244 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:240 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:236 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:232 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:228 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:224 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:220 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:216 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:232 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:228 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:224 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:220 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:216 ; GFX11-TRUE16-NEXT: scratch_load_b32 v103, off, s32 offset:388 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:8 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:16 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:24 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:32 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:40 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:8 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:16 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:24 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:40 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v86, off, s32 offset:48 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:56 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:64 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:72 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:80 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:88 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:112 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:64 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:72 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:80 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:88 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:96 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:104 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:112 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:120 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v160, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:136 @@ -87097,146 +87130,144 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v164, off, s32 offset:192 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:200 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v165, off, s32 offset:208 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:212 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:204 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:212 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:204 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:196 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:188 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:180 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:172 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:164 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:156 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 offset:148 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:140 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:124 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:116 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:108 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:100 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:180 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:172 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:156 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:148 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:140 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:124 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v97, off, s32 offset:84 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:76 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v102, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v113, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v114, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v112, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:12 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v115, off, s32 offset:4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v30.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.h, v26.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.h, v24.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v6.l +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v116, off, s32 offset:4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v26.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.h, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v150.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v151.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v148.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.l, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v23.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.l, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v50.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v50.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v49.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v54.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v53.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v64.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(61) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(54) ; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v103 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(53) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v80.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(52) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v81.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.h, 8, v82.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v82.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(50) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v82.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v83.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(49) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v83.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v84.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(48) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.h, 8, v86.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v119.h, 8, v85.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.l, 8, v86.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v86.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(46) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.h, 8, v87.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(45) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v87.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v114.h, 8, v96.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(44) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v96.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v115.h, 8, v97.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v98.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(42) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.l, 8, v99.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.h, 8, v99.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(41) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v99.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(40) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v100.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v101.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v101.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(38) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v160.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v160.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(37) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v160.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v160.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(36) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v161.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v161.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v161.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v161.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(34) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v162.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v162.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v162.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(32) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v163.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v163.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v163.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.l, 8, v163.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v164.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v164.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.l, 8, v164.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v164.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(28) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v165.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v165.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.l, 8, v165.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.l, 8, v69.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.l, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v55.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v55.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v54.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v54.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v52.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v49.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -87250,720 +87281,746 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB58_3: ; %cmp.false -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v149.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v149.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v151.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v148.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v148.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v150.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v150.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v151.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v151.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v145.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v144.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v145.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v144.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v146.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v132.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v135.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v146.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v147.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v131.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v132.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v147.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v119.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v133.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v119.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v130.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v133.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v134.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v118.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v134.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v114.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v128.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v150.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v146.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v151.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v150.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v147.h +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v0.h, v149.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v149.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v144.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v133.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v148.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v145.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v1.h, v147.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v129.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v130.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v113.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v116.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v129.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v102.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v101.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v116.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v117.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v117.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v118.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v98.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v97.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v134.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v2.l, v146.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v131.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v135.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v3.l, v145.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v128.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v118.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v116.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v4.l, v135.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v129.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v134.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v5.l, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v132.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v6.l, v132.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v130.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v112.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v7.l, v130.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v129.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v102.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v97.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v96.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v103.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v112.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v85.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v112.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v113.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v85.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v84.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v83.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v99.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v100.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v80.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v100.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v101.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v80.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v71.h -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v70.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v8.l, v128.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v112.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v119.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v100.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v9.l, v118.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v102.h +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v117.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v97.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v10.l, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v100.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v115.h +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v87.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v11.l, v114.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v11.h, v113.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v85.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v12.l, v113.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v103.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v83.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v71.l -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v86.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v96.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v67.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v87.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v67.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v66.h -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v81.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v82.l -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v82.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v83.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v52.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v48.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v69.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v13.l, v103.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v101.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v81.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v14.l, v101.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v84.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v99.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v71.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v15.l, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v81.h +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v15.h, v98.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v69.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v16.l, v97.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v71.h +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v16.h, v96.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v67.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v68.h -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v69.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v70.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v39.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v55.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v64.h -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v65.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v17.l, v87.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v69.h +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v17.h, v86.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v64.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v18.l, v85.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v67.h +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v18.h, v84.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v19.l, v83.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v65.h +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v19.h, v82.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v20.l, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v20.h, v80.h +; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v21.l, v80.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v21.h, v70.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v53.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v50.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v22.l, v70.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v22.h, v68.h +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v23.l, v68.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v23.h, v66.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v24.l, v66.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v24.h, v65.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v27, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v25.l, v64.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v25.h, v55.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v28, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v26.l, v55.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v26.h, v54.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v27.l, v54.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v28.h, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v28.l, v53.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v28.h, v52.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v33.l, v29.h, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v29.l, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v30.l, v50.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v32.l, v49.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB58_2 ; GFX11-TRUE16-NEXT: .LBB58_4: ; %cmp.true -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v149.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v149.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v148.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v148.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v146.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v147.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v150.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v150.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v151.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v145.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v151.l, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v144.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v151.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v150.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v149.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v149.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v147.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v134.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v133.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v144.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v135.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v135.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v148.l, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v146.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v147.h, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v147.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v132.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v131.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v131.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v119.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v132.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v133.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v6 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v130.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v145.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v131.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v145.h, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v131.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v119.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v118.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v133.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v134.h, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v134.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v128.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v128.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v128.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v135.h, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v135.l, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v129.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.l, v32.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v134.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v118.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v133.l, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v119.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v132.h, v6.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v114.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v114.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v115.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v113.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v116.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v132.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v117.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v130.h, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v114.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v130.l, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v129.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v130.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v116.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v116.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v102.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v101.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v102.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.l, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v112.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v128.l, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v112.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v98.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v117.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v119.h, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v102.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l ; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v118.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v102.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v117.h, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v116.l, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v100.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v115.h, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v97.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v114.h, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v103.l, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v117.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v97.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v103.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v97.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v96.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v85.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v112.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v85.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v112.h, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v113.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v99.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v84.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v99.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v84.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v83.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v80.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v113.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v87.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v113.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v96.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v103.h, v13.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v85.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v103.l, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v86.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v13.h ; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v100.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v80.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v100.h, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v101.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v83.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l ; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v101.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v84.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v99.h, v15.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v81.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v99.l, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, v81.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v15.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v98.l, v16.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v71.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v97.l, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v71.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v86.l, v17.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v71.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v70.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v86.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v71.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v67.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v87.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v96.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v67.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v16.h ; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v87.h, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v81.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v66.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v66.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v52.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v81.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v96.h, v17.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v69.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v87.l, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v69.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v86.h, v18.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v67.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v85.l, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v67.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v19.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v18.h ; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v52.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v82.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v82.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v49.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v48.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v84.h, v19.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v64.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.h, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v65.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v19.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v82.h, v20.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v51.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v82.l, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, v52.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v21.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v80.h, v21.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v80.l, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v68.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v69.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v68.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v69.h, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v39.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v70.l, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v38.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v37.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v21.h ; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v70.h, v22.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v70.l, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v55.h, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v64.h, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v64.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v65.l, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v22.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v68.h, v23.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v68.l, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v23.h ; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v65.h, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v36.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v66.l, v24.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v66.h, v24.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v64.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v26, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v65.l, v25.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v25.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v26.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v27, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v55.l, v26.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v55.h, v26.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v26.h ; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v54.h, v27.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v54.l, v27.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v53.l, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v54.l, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v55.l, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v53.h, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v54.h, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v34.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v33.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v53.h, v28.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v33.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v49.h, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v50.l, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v50.h, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v51.l, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v51.h, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v53.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v52.h, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v50.h, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v51.l, v30.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v50.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.l, 0x300, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v30.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v49.h, v30.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v32.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -111743,64 +111800,64 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) @@ -111828,50 +111885,50 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[65:66], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[54:55], 24, v[7:8] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] @@ -111915,50 +111972,50 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[66:67], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 24, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v27 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v26 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v24 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v20 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v17 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v17 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v1 ; GFX11-TRUE16-NEXT: .LBB72_4: ; %end @@ -111966,319 +112023,314 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v162.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v1.h, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v161.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v1.h, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v2.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff, v66 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v55, v39 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v67 -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v65 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v2.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v66, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v149.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v39, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v65 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v66 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v148.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v5.l, v33.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v147.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v39, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v33.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v55, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v64 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v55, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v144.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v39, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v54, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v54, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v53 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v64 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v53, v55 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v9.l, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v132.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v39, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v53, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v52, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v129.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v128.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v53 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v52, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v51, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v51, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v50, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v113.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v49, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v112.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v18.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v50 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v49, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v102.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v19.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v48, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v20.h, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v48, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v49 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v21.l, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v98.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v22.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v38, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v23.h, v24.l ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v39.l, v12.h, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v55, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v119.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v117.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v16.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v114.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v113.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v112.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v103.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v101.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v96.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v85.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v82.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v80.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v71.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v69.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v65 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v53, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v39, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v38, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v35, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v37, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v25.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v87.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v86.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v24.l, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v37, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v85.l +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v25.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v84.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v36, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v83.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v26.h, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v36, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v27.l, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v34.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v28.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v71.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v29.h, v30.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v31.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v69.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v30.l, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v31.l, v31.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v32.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v68.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v39 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v39.h, v32.l, v32.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v33, v39 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v16f64_to_v128i8: @@ -121787,63 +121839,63 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:384 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:380 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:376 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:372 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:368 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:364 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:372 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:368 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:364 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:360 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:356 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v49, off, s32 offset:352 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:348 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:356 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v50, off, s32 offset:352 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:348 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:344 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:340 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:336 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:336 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:332 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:328 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:324 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:328 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:324 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v53, off, s32 offset:320 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:316 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:312 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:316 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:312 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:308 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v54, off, s32 offset:304 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:300 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:296 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:292 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:288 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:284 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:292 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:288 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:284 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:280 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:276 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v55, off, s32 offset:272 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:268 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:264 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32 offset:276 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:272 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:268 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:264 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v39, off, s32 offset:260 ; GFX11-TRUE16-NEXT: s_clause 0x1f -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:256 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:256 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v48, off, s32 offset:252 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:248 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:248 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v48, off, s32 offset:244 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v68, off, s32 offset:240 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:236 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:232 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:228 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:224 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v52, off, s32 offset:220 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:216 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:232 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v51, off, s32 offset:228 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:224 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:220 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:216 ; GFX11-TRUE16-NEXT: scratch_load_b32 v103, off, s32 offset:388 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:8 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:16 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:24 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:32 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:40 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v82, off, s32 offset:8 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v82, off, s32 offset:16 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:24 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:32 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:40 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v86, off, s32 offset:48 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v87, off, s32 offset:56 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:64 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:72 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:80 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:88 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:112 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:64 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:72 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:80 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v99, off, s32 offset:88 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v99, off, s32 offset:96 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v101, off, s32 offset:104 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:112 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v160, off, s32 offset:120 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v160, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v161, off, s32 offset:136 @@ -121857,146 +121909,144 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v164, off, s32 offset:192 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v165, off, s32 offset:200 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v165, off, s32 offset:208 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:212 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:204 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:212 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:204 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:196 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v67, off, s32 offset:188 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v70, off, s32 offset:180 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:172 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:164 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:156 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v80, off, s32 offset:148 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v83, off, s32 offset:140 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:132 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v84, off, s32 offset:124 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v85, off, s32 offset:116 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:108 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v96, off, s32 offset:100 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v97, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:180 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v69, off, s32 offset:172 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32 offset:164 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v71, off, s32 offset:156 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v81, off, s32 offset:148 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v81, off, s32 offset:140 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v83, off, s32 offset:132 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v84, off, s32 offset:124 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v85, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v86, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v87, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v96, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v97, off, s32 offset:84 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v98, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v101, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v98, off, s32 offset:76 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v100, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v100, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v102, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v102, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v113, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v114, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v112, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v112, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v114, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v115, off, s32 offset:12 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v115, off, s32 offset:4 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v30.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.h, v26.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.h, v24.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.l, v6.l +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v116, off, s32 offset:4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v26.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.h, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v150.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v151.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v151.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v150.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v147.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v149.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v146.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v148.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v145.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.l, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v135.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v133.l, 8, v23.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.l, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v134.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v132.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(62) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v50.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v50.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v49.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v54.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v53.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.h, 8, v64.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v64.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v55.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.l, 8, v69.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(61) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v68.h -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(59) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(54) ; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v103 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(53) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v80.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(52) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v81.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.h, 8, v82.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(51) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v128.l, 8, v82.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(50) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v130.l, 8, v82.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v129.l, 8, v83.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(49) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v83.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v84.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(48) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.h, 8, v86.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v119.h, 8, v85.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(47) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.l, 8, v86.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v116.l, 8, v86.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(46) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v117.h, 8, v87.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(45) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v118.l, 8, v87.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v114.h, 8, v96.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(44) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v96.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v115.h, 8, v97.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(43) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v98.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(42) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.l, 8, v99.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.h, 8, v99.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(41) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v112.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.l, 8, v99.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(40) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v113.l, 8, v100.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v103.h, 8, v101.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(39) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v101.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v101.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(38) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v160.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.h, 8, v160.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(37) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.l, 8, v160.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.l, 8, v160.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(36) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v100.h, 8, v161.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v99.h, 8, v161.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(35) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v101.l, 8, v161.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v97.l, 8, v161.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(34) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v98.l, 8, v162.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v162.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v162.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(32) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.l, 8, v163.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.h, 8, v163.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v87.h, 8, v163.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v85.l, 8, v163.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(30) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v96.l, 8, v164.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v86.h, 8, v164.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.l, 8, v164.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.h, 8, v164.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(28) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v81.h, 8, v165.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v84.h, 8, v165.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.l, 8, v165.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v83.l, 8, v69.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v69.h, 8, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v82.h, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.l, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v80.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.l, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v70.h, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.l, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v68.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v66.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v64.l, 8, v55.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.l, 8, v55.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v65.h, 8, v54.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.l, 8, v54.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v55.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v53.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.l, 8, v52.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v51.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.l, 8, v49.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -122010,720 +122060,746 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB74_3: ; %cmp.false -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v149.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v149.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v151.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v148.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v148.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v150.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v150.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v151.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v151.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v145.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v144.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v145.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v144.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v135.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v146.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v132.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v135.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v146.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v147.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v131.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v132.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v147.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v131.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v119.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v133.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v119.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v130.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v133.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v134.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v118.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v134.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v114.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v128.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v150.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v146.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v151.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v150.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v147.h +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v0.h, v149.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v149.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v144.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v133.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v148.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v144.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v145.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v1.h, v147.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v115.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v129.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v130.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v113.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v116.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v129.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v102.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v101.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v116.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v117.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v8.h, v117.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v118.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v98.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v97.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v103.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v134.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v2.l, v146.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v131.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v135.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v3.l, v145.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v128.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v118.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v116.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v4.l, v135.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v129.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v134.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v5.l, v133.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v132.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v114.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v6.l, v132.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v130.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v112.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v7.l, v130.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v115.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v129.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v102.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v97.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v96.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v103.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v112.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v85.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v112.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v113.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v85.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v84.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v99.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v83.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v99.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v100.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v80.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v100.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v101.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v80.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v71.h -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v70.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v8.l, v128.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v112.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v119.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v100.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v9.l, v118.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v102.h +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v117.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v97.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v10.l, v116.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v100.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v115.h +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v87.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v11.l, v114.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v98.h +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v11.h, v113.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v85.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v12.l, v113.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v103.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v83.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v71.l -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v86.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v87.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v96.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v67.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v87.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v67.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v66.h -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v17.h, v81.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v82.l -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v18.h, v82.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v83.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v52.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v48.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v68.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v69.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v13.l, v103.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v86.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v101.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v81.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v14.l, v101.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v84.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v99.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v71.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v15.l, v99.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v81.h +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v15.h, v98.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v69.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v16.l, v97.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v71.h +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v16.h, v96.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v67.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v68.h -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v69.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v70.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v39.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v55.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v64.h -; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v22.h, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v23.h, v65.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v17.l, v87.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v69.h +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v17.h, v86.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v64.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v18.l, v85.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v67.h +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v18.h, v84.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v19.l, v83.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v65.h +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v19.h, v82.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v20.l, v82.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v20.h, v80.h +; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v21.l, v80.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v21.h, v70.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v53.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v27.l, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v27.h, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v50.h -; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v28.h, v51.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v51.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v22.l, v70.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v22.h, v68.h +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v23.l, v68.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v23.h, v66.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v24.l, v66.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v24.h, v65.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v27, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v25.l, v64.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v25.h, v55.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v28, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v26.l, v55.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v26.h, v54.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v27.l, v54.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v35.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v28.h, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v34.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v28.l, v53.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v28.h, v52.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v33.l, v29.h, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v29.l, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v30.l, v50.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v32.l, v49.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB74_2 ; GFX11-TRUE16-NEXT: .LBB74_4: ; %cmp.true -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v149.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v149.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v148.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v145.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v151.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v148.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v150.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v146.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v147.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v150.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v150.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v151.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v145.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v151.l, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v144.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v151.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v150.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v149.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v149.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v147.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v134.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v133.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v144.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v144.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v135.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v135.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v146.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v148.l, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v146.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v147.h, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v147.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v132.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v131.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v131.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v119.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v132.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v133.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v6 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v130.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v145.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v131.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v145.h, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v131.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v119.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v118.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v133.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v134.h, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v134.l, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v128.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v128.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v11, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v128.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v135.h, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v135.l, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v129.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v32.l, v32.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v134.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v118.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v133.l, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v119.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v132.h, v6.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v114.h, 3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v114.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v115.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v113.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v116.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v132.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v117.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v130.h, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v114.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v130.l, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v115.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v129.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v130.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v116.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v116.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v102.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v101.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v102.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v129.l, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v112.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v128.l, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v112.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v98.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v117.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v119.h, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v102.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l ; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v118.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v102.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v117.h, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v100.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v116.l, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v100.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v115.h, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v97.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v114.h, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v98.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v103.l, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v117.h, v8.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v97.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v103.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v97.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v96.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v85.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v112.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v85.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v112.h, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v113.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v99.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v84.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v99.h, v12.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v84.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v83.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v17, v18 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v80.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v113.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v87.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v14, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v113.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v96.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v103.h, v13.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v85.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v103.l, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v86.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v13.h ; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v100.l, v13.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v80.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v100.h, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v101.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.h, v83.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l ; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v101.l, v14.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v84.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v99.h, v15.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.h, v81.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v99.l, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, v81.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v16.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v15.h +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v98.l, v16.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v71.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v18, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v97.l, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v71.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v86.l, v17.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v16.l, 0x300, v13.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v19, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v71.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v70.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v86.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v21, v22 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v71.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, v67.h, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v87.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v96.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.h, v67.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v16.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v16.h ; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v87.h, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v81.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v16, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v66.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v66.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v52.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v81.h, v17.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v96.h, v17.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.h, v69.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v19, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v87.l, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, v69.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v18.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v17.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v86.h, v18.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.h, v67.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v20, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v85.l, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, v67.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v19.h, 0xff, v19.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v18.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v18.h ; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, 0x300, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v22, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v52.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v82.l, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v82.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.l, v19.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v49.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v48.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v18.l, 0x300, v18.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v18.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v19.l, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v84.h, v19.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.h, v64.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v21, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v83.h, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v20.l, v65.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v19.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, 0x300, v19.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v82.h, v20.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, v51.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v82.l, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.l, v52.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v21.h, 0xff, v21.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v20.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v80.h, v21.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v23, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v80.l, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v68.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v69.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v24, v25 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v68.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v69.h, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v21, v29 -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, v39.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v70.l, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.h, v38.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v37.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v21.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v21.h ; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.h, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v70.h, v22.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v24, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v70.l, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, v48.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v55.h, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v64.h, v23.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v20, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v64.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v65.l, v23.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v22.l, 0x300, v22.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v23.l, 0x300, v23.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v22.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v22.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v68.h, v23.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v25, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v68.l, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, v39.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v23.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v23.h ; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v22.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, 0x300, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v29, v30 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v65.h, v24.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v36.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v66.l, v24.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v26, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v66.h, v24.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v24.l, 0x300, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v64.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v26, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v65.l, v25.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v26.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v27 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v25.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v25.l, 0x300, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v26.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v27, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v55.l, v26.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v55.h, v26.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v26.h ; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v29, v31 +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v54.h, v27.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v54.l, v27.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v53.l, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v54.l, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v55.l, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v53.h, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v54.h, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v35.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v31, v37 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v26, v39 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v29 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, v34.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, v33.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v27.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v53.h, v28.h ; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v28.h, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v29, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.h, v33.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v49.h, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v50.l, v27.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v50.h, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v51.l, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v51.h, v29.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v25, v38 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v31 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.l, 0x300, v27.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, 0x300, v27.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v28.l, 0x300, v28.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.l, 0x300, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v53.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v30.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v52.h, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v50.h, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v29.l, 0x300, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v36 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v37, v38 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v39, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v33 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v31, v34 +; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v51.l, v30.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v50.l, v30.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v33.l, 0x300, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v30.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v31 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v30.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v33 +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v49.h, v30.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v31 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v31.h, 0x300, v32.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v31 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -160055,116 +160131,116 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:8 ; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4 ; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) ; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v33 @@ -160187,341 +160263,338 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[52:53], 24, v[11:12] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[67:68], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[23:24] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v167, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 24, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v166, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v167, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v62, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v59, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 8, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v91, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 8, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v108, 8, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v28 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v27 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 24, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v166, 24, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v41, 8, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v44, 8, v21 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v47, 24, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 24, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 8, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 8, v21 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 24, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 8, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v62, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v17 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[68:69], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[21:22] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[53:54], 24, v[17:18] -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.h, v2.h -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.h, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.h, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v4.h ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v180.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.h, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.h, v6.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v74.h, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.h, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v161.h, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.h, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.h, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.h, v8.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.h, v9.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.h, v10.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.h, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v59.h, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.h, v12.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v94.h, v13.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v76.h, v14.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.h, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.h, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v104.h, v13.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v78.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v77.h, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v95.h, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v93.h, v16.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v92.h, v16.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v17.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.h, v18.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v19.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.h, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v19.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.h, v20.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v21.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v22.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.h, v23.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v24.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.h, v24.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v21.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.h, v23.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v24.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.h, v25.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.h, v26.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.h, v26.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.h, v26.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.h, v26.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.h, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.h, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.h, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.h, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.h, v28.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.h, v30.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.h, v30.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.h, v31.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v32.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.h, v32.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.h, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v32.h ; GFX11-TRUE16-NEXT: .LBB90_2: ; %Flow ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB90_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v17 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v18 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_lshlrev_b32 v18, 16, v18 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v20 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v33, 16, 1 ; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v18, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v18 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 -; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v33, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v37, v37, v18, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v37, v39, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v70, v37, v39 :: v_dual_add_f32 v33, 0x40c00000, v33 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v33, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v17, 16, 1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v55.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v70, v36, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v35, 0x40c00000, v35 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v37, v50, v17, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20 +; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v33, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v70.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v36, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_lshlrev_b32 v17, 16, v17 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17 ; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v34, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v34 -; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v80, v37, v51, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v17, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v17 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 ; GFX11-TRUE16-NEXT: v_add3_u32 v18, v48, v34, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35 +; GFX11-TRUE16-NEXT: v_add3_u32 v37, v50, v17, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v71, v37, v51 :: v_dual_lshlrev_b32 v20, 16, v20 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_add_f32 v20, 0x40c00000, v20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v71.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v20 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v80.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v49, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v33, v70 +; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v33, v55 ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v20, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35 ; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v34, v17 ; GFX11-TRUE16-NEXT: v_add3_u32 v34, v36, v35, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 24, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v18 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v17 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 +; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v62, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v19, 0x40c00000, v19 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v33, v37, vcc_lo -; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v19, 16, v19 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v22 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v36, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v71, v34, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v81.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v19, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v36, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v80, v34, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v19 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22 +; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v19, 0x7fff ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19 ; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v36, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v19, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v83, v33, v37 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v81.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v17 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v84, v33, v37, vcc_lo ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v22, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v83.h +; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v35 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v84.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v20, v38, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v35, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v20, v33, v22, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v22 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v82, v20, v33 :: v_dual_add_f32 v35, 0x40c00000, v35 -; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v34, v71 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v82.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v35, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v35 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v47, 24, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v20 ; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v35, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v38 :: v_dual_cndmask_b32 v84, v19, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v83, v20, v33, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 +; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v34, v80 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v19, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v37, v36 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v24 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v21 -; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v83.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 24, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 8, v20 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24 +; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v82 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 8, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v22 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v21 +; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v21, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v33, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36 ; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v21, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v84 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v19 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v34, v37, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v86, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v23 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24 +; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v24, 16, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v36, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v41, 8, v22 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v35, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v24, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v24, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v24 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v36, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v85.h -; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v24, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v35, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v86, v34, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v86.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v34, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v26 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v23 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v37 :: v_dual_lshlrev_b32 v26, 16, v26 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v33, v39, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v86.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v23, 0x40c00000, v23 :: v_dual_lshlrev_b32 v26, 16, v26 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 +; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v35, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26 ; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v23, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v37, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v33, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v23 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 8, v21 ; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v23, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v37, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v44, 8, v21 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v96, v34, v36, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v97, v34, v36, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v37 :: v_dual_add_f32 v34, 0x40c00000, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v97.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v37, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v38 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v96.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v24, v39, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34 +; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v37, 0x7fff ; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1 -; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v33, v87 -; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v26, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v24, v39, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v26 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v36, v23 -; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v26, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v87.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v23 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v166, 24, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 8, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v97, v33, v37, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v25, 0x40c00000, v25 +; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v33, v85 +; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v26, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 24, v24 +; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v26, 0x7fff ; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v36, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 8, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v98, v35, v38, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 8, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v98, v33, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 +; GFX11-TRUE16-NEXT: v_add3_u32 v26, v26, v36, 0x7fff +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v96, v35, v38 :: v_dual_add_f32 v25, 0x40c00000, v25 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v28 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v98.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v25, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v25 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_add_f32 v28, 0x40c00000, v28 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25 -; GFX11-TRUE16-NEXT: v_add3_u32 v26, v26, v36, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v25, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v101, v33, v37, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v97.h -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v36, v26, v38 :: v_dual_add_f32 v35, 0x40c00000, v35 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v27 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v101.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v35, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v28, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v35 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v101, v33, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v28, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 ; GFX11-TRUE16-NEXT: v_add3_u32 v25, v25, v35, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v26, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v26, v33, v28, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v99, v26, v33, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v27 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v101.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v100, v26, v33, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 -; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v34, v98 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v100, v25, v39, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v27, 0x40c00000, v27 +; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v34, v96 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v100.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v99, v25, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v37, v36 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v30 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v99.h -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 24, v26 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v27, 0x40c00000, v27 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v100 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36 ; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v27, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v27 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36 +; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 24, v28 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v28 ; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v27, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v25 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v112, v34, v37, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v33, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v112, v34, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v36, 16, 1 +; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v99 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v26 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v35, v38, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v36, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v29 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29 @@ -160529,21 +160602,22 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v112.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v28 ; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v30, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v25 ; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v35, v27 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v103, v34, v38 :: v_dual_and_b32 v38, 0xffff0000, v32 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v29 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v29, 16, 1 ; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v102, v33, v39 :: v_dual_add_f32 v37, 0x40c00000, v37 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v29 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v29, 0x7fff ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v103.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v37, 16, 1 @@ -160556,45 +160630,44 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v113.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v30, v39, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1 ; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v33, v102 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34 -; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v36, v29 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31 ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v32, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32 +; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v36, v29 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v32 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v31, 0x40c00000, v31 ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v32, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 24, v30 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v36, 16, 1 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v31 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v29 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v33, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v115, v33, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v31, 16, v31 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v115.h +; GFX11-TRUE16-NEXT: v_add_f32_e32 v31, 0x40c00000, v31 +; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v36, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v35, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v31, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 ; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v36, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v115, v35, v38, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v31, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v114.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v116, v33, v37, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v116, v33, v37 :: v_dual_and_b32 v35, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v2, 16, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v116.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v31, v35, 16, 1 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v32, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v1 @@ -160607,10 +160680,10 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_add3_u32 v32, v33, v2, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v131, v32, v33, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v133, v32, v33, vcc_lo ; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 -; GFX11-TRUE16-NEXT: v_bfi_b32 v32, 0xffff, v34, v115 +; GFX11-TRUE16-NEXT: v_bfi_b32 v32, 0xffff, v34, v114 ; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v132, v31, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v37, v36 @@ -160622,9 +160695,9 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v33, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v2.l, v131.h +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v2.l, v133.h ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 24, v32 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v144, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v146, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v3 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 @@ -160640,252 +160713,240 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v36, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v37, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v147, v34, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v148, v34, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v3, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v36, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v35.l, v144.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v37, 0x7fff -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 24, v2 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v149, v33, v39, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37 +; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v144, v33, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v35, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v108, 8, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v162, v34, v36, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37 +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v33.l, v148.h +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v35.l, v146.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v164, v34, v36, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v36.l, v162.h +; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v38 +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v35, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v36.l, v164.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v39, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v7 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34 ; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v36, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v33.l, v147.h -; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v38 :: v_dual_lshlrev_b32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 8, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v33, v149 -; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v4 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 +; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v33, v144 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 8, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 8, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v6, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v164, v33, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v108, 8, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v32 +; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v36, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v165, v33, v37, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36 ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v5, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v165, v35, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v36, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v36, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v34.l, v165.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v161, v35, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v5, 0x7fff ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v35 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v36, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v34.l, v164.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v180, v33, v37, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v180, v33, v37 :: v_dual_add_f32 v35, 0x40c00000, v35 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v35, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e64 v37.l, v180.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v6, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v35, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v35, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v6, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v35, 0x7fff ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v33, v8, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v178, v6, v33, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v179, v6, v33, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v39 ; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v7, 16, 1 -; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v34, v165 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v179, v5, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v34, v161 +; GFX11-TRUE16-NEXT: v_mov_b16_e64 v8.l, v179.h +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v178, v5, v38 :: v_dual_add_f32 v33, 0x40c00000, v39 ; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v37, v36 -; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v33, 16, 1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v9 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v10 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v33, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v7 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v33, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v8.l, v178.h -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v46, v35, v37, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36 -; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v179 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v34, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33 +; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v33, 0x7fff +; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v178 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v47, v35, v37, vcc_lo ; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v36, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v36 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v62, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 8, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v59, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 8, v8 ; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v36, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v45, v7, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v34, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v47.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v91, 8, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v7, v37, vcc_lo ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v39 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v46.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v9, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v9, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v9, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v10, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v10 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v10, 0x7fff -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v42, v35, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v41, v35, v38 :: v_dual_lshlrev_b32 v10, 16, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v45.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v44.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v37, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v35, v42 -; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v37 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v35, v41 +; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v37, 0x7fff -; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v12, 16, 1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v51 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 24, v10 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v7, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 24, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v59, v38, v50, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v61, v38, v50 :: v_dual_add_f32 v12, 0x40c00000, v12 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_add3_u32 v48, v48, v12, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v14 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v59.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v74, v35, v49, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v61.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v12, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v73, v35, v49, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v48, v48, v12, 0x7fff ; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_lshlrev_b32 v11, 16, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 8, v10 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v60, v48, v52 :: v_dual_add_f32 v37, 0x40c00000, v51 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v57, v48, v52, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 ; GFX11-TRUE16-NEXT: v_bfe_u32 v49, v14, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v7, v60 -; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v7, v57 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v36, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v11 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v74.h ; GFX11-TRUE16-NEXT: v_add3_u32 v11, v35, v37, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v37 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v73.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v7, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 24, v12 -; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v36, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v167, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 8, v12 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v35, vcc_lo ; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38 ; GFX11-TRUE16-NEXT: v_add3_u32 v37, v39, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v7 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v35, 16, 1 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[64:65], 24, v[9:10] +; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v36, v9 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v39 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v89, v37, v38, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35 ; GFX11-TRUE16-NEXT: v_add3_u32 v37, v48, v35, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v9 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v76, v37, v38 :: v_dual_and_b32 v37, 0xffff0000, v16 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v39 :: v_dual_lshlrev_b32 v16, 16, v16 ; GFX11-TRUE16-NEXT: v_add3_u32 v39, v49, v14, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v14 ; GFX11-TRUE16-NEXT: v_bfe_u32 v49, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v77, v37, v38, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v77, v39, v48, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: v_add3_u32 v14, v49, v7, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v77.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v78, v39, v48, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[64:65], 24, v[9:10] +; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v78.h ; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v14, v35 :: v_dual_add_f32 v14, 0x40c00000, v37 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v37 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v37 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v15, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v13, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13 ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v16, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v94, v35, v39, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v37, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v37 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v104, v35, v39, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v16, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v37, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v37 +; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v37, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v93, v13, v49, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 ; GFX11-TRUE16-NEXT: v_add3_u32 v35, v48, v14, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v14 -; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v37, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v15, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v91, v13, v49, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37 -; GFX11-TRUE16-NEXT: v_add3_u32 v13, v50, v15, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v94.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v104, v39, v51, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v95, v39, v51, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v104.h +; GFX11-TRUE16-NEXT: v_add3_u32 v13, v50, v15, 0x7fff ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v89.h -; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v38, v76 +; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v38, v77 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v92, v35, v48, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v104.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v91.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v95.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v93.h ; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v39, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v14 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v166, 8, v14 ; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v35, v92 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[52:53], 24, v[11:12] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[53:54], 24, v[17:18] @@ -160905,332 +160966,327 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[68:69], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[21:22] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v167, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v43, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v7 ; GFX11-TRUE16-NEXT: .LBB90_4: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v144.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v146.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v108.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v132.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v106.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v133.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v107.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v149.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.h, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v131.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v107.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v68.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v132.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v2.h, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v106.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v78.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v105.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v162.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v164.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v1.h, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v105.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v94.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v91.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v4.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v148.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v4, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v147.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v95.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v93.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v5.l, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v180.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v90.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v66.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v164.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v180.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v90.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v144.h ; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v88.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v165.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v8, v10 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v14 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v4.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v165.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v88.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v8, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v5.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v47.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v76.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v58.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v14 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v75.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v161.h +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v6.h, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v179.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v72.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v6.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v7.l ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v65.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v46.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v75.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v178.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v63.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v179.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v62.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v74.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v57.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v8.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v73.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v178.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v59.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v14 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v56.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v8.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v44.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v45.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v56.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v43.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v89.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v41.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v42.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v16, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v10.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v10, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v89.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v40.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v61.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v183.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v16, v14 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v42.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v43.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v60.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v181.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v11.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v104.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v176.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v166.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v16, v14 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v167.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v57.h +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v12.h, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v78.h ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v94.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v167.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v59.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v182.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v77.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v163.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v20, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v76.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v104.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v14 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v91.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v92.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v135.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v55.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v73.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v80.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v79.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v70.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v72.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v18 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v83.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v61.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v12.l, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v16, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v77.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v95.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v64, v18, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v93.h +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v149.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v65, v18, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v71.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v79.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v92.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v134.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v13.h, v15.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v66, v18, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v70.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v74.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v67, v18, v14 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v46.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v15.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v84.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v63.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v16, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v55.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v62.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v81.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v60.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v71.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v47.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v81.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v58.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v20, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v86.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v80.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v45.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v19, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v83.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v40.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v19, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v38.l ; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v85.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v44.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v18, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v96.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v177.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v97.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v182.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.h, v19.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v13.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v82.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v41.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v84.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v183.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v86.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v176.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v181.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v13.h, v19.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v22, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v87.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v177.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v22, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v147.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v13.h, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v21.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v101.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v163.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v22, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v85.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v162.l ; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v87.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v166.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v101.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v97.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v98.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v148.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v24, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v112.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v145.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v99.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v100.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v113.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v103.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v28, v19 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v102.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v116.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v98.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.h, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v13.h, v22.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v26, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v28 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v112.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v23.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v96.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v145.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v25, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v24.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v13.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v100.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v135.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v25, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v113.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v131.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v26 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.h, v25.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v99.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v13.h, v25.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v28, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v30 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v103.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v26.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v129.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v28, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v30 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v13.h, v27.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v27.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v116.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v128.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v27.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v28, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v102.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.l, 8, v119.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v115.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.h, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v13.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v114.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v115.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v117.l -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v13.h, v28.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v32, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v34 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v114.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v30 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v31, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v30, v14 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[64:67], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[15:18], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[19:22], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[23:26], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[27:30], off offset:112 ; GFX11-TRUE16-NEXT: s_clause 0x1f ; GFX11-TRUE16-NEXT: scratch_load_b32 v108, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_b32 v107, off, s32 offset:16 @@ -185249,64 +185305,64 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) @@ -185331,52 +185387,52 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[52:53], 24, v[11:12] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[67:68], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[21:22] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 8, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 24, v28 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v27 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[68:69], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20] @@ -185436,371 +185492,366 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[21:22] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[53:54], 24, v[17:18] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 8, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 24, v28 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v27 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v17 ; GFX11-TRUE16-NEXT: .LBB94_4: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v163.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v162.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v51.l, v1.h, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v68.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v161.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v1.h, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v2.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff, v68 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v54, v51 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v68 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v69 -; GFX11-TRUE16-NEXT: v_or_b16 v51.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v67 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v2.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v68, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v149.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v66.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v51, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v67 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v68 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v67 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v148.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v5.l, v33.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v147.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v51, v54 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v65.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v67 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v33.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v51, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v66 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v144.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v66 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v132.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v66 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v65 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v54, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v131.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v65 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v66 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v64, v65 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v51, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v119.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v116.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v112.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v52, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v102.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v52, v51 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v49, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v49, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v84.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v48, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v39, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v39, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v134.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v133.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v18.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v129.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v39, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v19.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v128.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v39, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v20.h, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v39, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v115.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v21.l, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v114.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v37.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v38, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v22.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v113.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v38, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v23.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v103.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v115.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v114.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v51.l, v12.h, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v101.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v65, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v51 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v87.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v16.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v85.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v145.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v144.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v132.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v113.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v112.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v96.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v82.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v80.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v71.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v67 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v54, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v39, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v38, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v35, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v37, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v25.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v98.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v24.l, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v37, v51 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v25.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v86.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v36, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v85.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v26.h, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v36, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v83.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v27.l, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v34.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v28.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v81.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v80.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v29.h, v30.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v31.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v71.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v30.l, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v51 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v31.l, v31.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v32.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v55.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v32.l, v32.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v33, v51 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v64f16_to_v128i8: @@ -208007,64 +208058,64 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2) @@ -208089,52 +208140,52 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[52:53], 24, v[11:12] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[67:68], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[21:22] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 8, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 24, v28 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v27 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[68:69], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20] @@ -208194,371 +208245,366 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[21:22] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[53:54], 24, v[17:18] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 24, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v15 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 24, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v11 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 8, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v146, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 24, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v150, 8, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v151, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 8, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v32 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v32 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v31 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v30 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v31 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v30 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v30 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 8, v29 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 24, v28 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v29 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v28 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 8, v28 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v27 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v26 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v24 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v23 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 24, v22 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 8, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 24, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v26 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v25 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v24 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 8, v23 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v114, 24, v22 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v22 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v21 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v20 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v18 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v133, 8, v20 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 8, v19 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 24, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v18 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v17 ; GFX11-TRUE16-NEXT: .LBB98_4: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v163.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v162.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v68.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v161.l -; GFX11-TRUE16-NEXT: v_or_b16 v51.l, v1.h, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v162.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v68.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v161.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v1.h, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v2.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v160.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v160.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v151.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff, v68 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v54, v51 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v68 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v69 -; GFX11-TRUE16-NEXT: v_or_b16 v51.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v151.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v67 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v2.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v150.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v68, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v149.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v66.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v51, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v67 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v68 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v67 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v66.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v148.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v5.l, v33.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v147.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v51, v54 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v33.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v65.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v67 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v33.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v51, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v66 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v146.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v144.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v66 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v65.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v132.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v66 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v7.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v134.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v65 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v54 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v131.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v8.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v54, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v130.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v131.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v65 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v66 -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v64, v65 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v119.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v65 +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v118.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v10.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v51, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v119.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v116.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v54, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v64 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v11.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v112.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v11.h, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v52, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v102.l +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v54 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v52, v51 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v99.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v13.l, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v49, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v14.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v96.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v49, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff, v52 +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v84.l +; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v48, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v17.h +; GFX11-TRUE16-NEXT: v_and_b16 v17.h, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v145.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v39, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v17.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v135.l +; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v39, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v134.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v133.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v19.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v18.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v19.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v129.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v39, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v19.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v20.h, 0xff, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v128.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v39, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v20.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v117.l +; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v20.h, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v39, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v115.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v48 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v21.l, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v114.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v37.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v38, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v22.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_and_b16 v23.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v113.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v38, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v39 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v23.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v23.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.l ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v103.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v115.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v114.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v51.l, v12.h, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v101.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v65, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v12, v51 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v87.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v16.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v85.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v145.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v144.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v133.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v129.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v16, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v132.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v128.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v15, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v113.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v118.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v17, v18 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v112.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v17 -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v96.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v17.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v20, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v22, v23 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v84.l -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v82.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v18.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v26, v27 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v80.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v71.l -; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v70.l -; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v19.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v66 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v67 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v20.l, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v21.l, v21.l, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v22.l, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v23.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v54, v64 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v39, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v38, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v35, v19 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v25, v26 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v27, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v29, v30 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v31, v32 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v33, v24 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v37, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v25.l, v25.l, v33.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v100.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v98.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v24.l, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v26.l, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v38 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v37, v51 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v87.l +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v25.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v26.l +; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v26.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v86.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v36, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v26.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v85.l +; GFX11-TRUE16-NEXT: v_or_b16 v27.l, v26.h, v27.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v36, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v83.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v37 +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v27.l, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v82.l +; GFX11-TRUE16-NEXT: v_or_b16 v29.l, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v34.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v35, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v28.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v29.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.l, 8, v81.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v35, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v36 +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v29.l, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v80.l +; GFX11-TRUE16-NEXT: v_or_b16 v30.l, v29.h, v30.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v34, v51 +; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v31.l, v33.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v71.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v70.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v30.l, v30.h +; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v32.l, v32.l, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v35 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v34, v51 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v31.l, v31.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v32.l +; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v55.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v33, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v51.h, v32.l, v32.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v33, v51 ; GFX11-TRUE16-NEXT: s_clause 0x5 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off offset:32 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:48 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:64 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:80 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:96 -; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:112 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[9:12], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[13:16], off offset:48 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[17:20], off offset:64 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[21:24], off offset:80 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[25:28], off offset:96 +; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[29:32], off offset:112 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v64i16_to_v128i8: diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll index 34d7ed9..3e96ab1 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll @@ -2675,79 +2675,76 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v10 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v9 :: v_dual_and_b32 v5, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v9 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v14, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v6 +; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2 +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v7, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0 ; GFX11-TRUE16-NEXT: .LBB22_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -4122,18 +4119,18 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -4147,107 +4144,103 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2 ; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -7140,79 +7133,76 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v10 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v9 :: v_dual_and_b32 v5, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v9 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v14, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v6 +; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2 +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v7, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0 ; GFX11-TRUE16-NEXT: .LBB46_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -8603,18 +8593,18 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -8628,107 +8618,103 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2 ; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -11253,79 +11239,76 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v10 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v9 :: v_dual_and_b32 v5, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v9 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v14, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v6 +; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2 +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v7, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0 ; GFX11-TRUE16-NEXT: .LBB66_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -12700,18 +12683,18 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -12725,107 +12708,103 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2 ; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -14952,79 +14931,76 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v10 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v9 :: v_dual_and_b32 v5, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v9 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v14, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v6 +; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2 +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v7, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0 ; GFX11-TRUE16-NEXT: .LBB82_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -16407,18 +16383,18 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -16432,107 +16408,103 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2 ; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -18254,83 +18226,83 @@ define <8 x i16> @bitcast_v8bf16_to_v8i16(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_and_b32 v1, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_add_f32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v8, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v3, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v12, v13, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 ; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v12, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v8, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v11, v12 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_add3_u32 v13, v14, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v13, v14, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v15, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v7 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.h ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v0, 16, v3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v1, 16, v2 -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v7, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v6, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v7, 16, v5 +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v4, 16, v6 ; GFX11-TRUE16-NEXT: .LBB94_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -19840,18 +19812,18 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -19865,107 +19837,103 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2 ; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -21172,79 +21140,79 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_and_b32 v5, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v1, v7, v9 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v1, v7 :: v_dual_and_b32 v7, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v11, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v11, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v7 :: v_dual_add_f32 v7, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v12, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v2, v2, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v14, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v13, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v7, v3 -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.h +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v7, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v5, v3 +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v9, v4 ; GFX11-TRUE16-NEXT: .LBB102_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -22758,18 +22726,18 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -22783,107 +22751,103 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2 ; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -23876,87 +23840,92 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v8 :: v_dual_and_b32 v3, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v14, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v8 :: v_dual_and_b32 v3, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v2, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v13, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v11 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v11 -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v12, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v10 -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v12, 0x7fff -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v9, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v2, v13, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v14, v9, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v15, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add3_u32 v0, v9, v13, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v8 :: v_dual_and_b32 v5, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v7, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v5, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v13, vcc_lo -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v8, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v10, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v15, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v14, v10, 0x7fff +; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v2, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v9, v13, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v1, v14 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v13, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v11 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v7, v2 -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v9, v1 +; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v9, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11] +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[2:3] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2 ; GFX11-TRUE16-NEXT: .LBB108_4: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v17.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h @@ -24976,18 +24945,18 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v15.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16 @@ -25001,107 +24970,103 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v9.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.h ; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v7.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v8.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v10 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v1.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v2.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v9 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2 ; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v9.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v9.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v12.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v7.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v7.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v8, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v9, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v7, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.h, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v4, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll index 2c78e34..5344095 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll @@ -659,7 +659,8 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 @@ -1132,7 +1133,8 @@ define half @bitcast_bf16_to_f16(bfloat %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll index a19567b..f8ffaa4 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll @@ -6296,33 +6296,32 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v19.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v17.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v13.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v22.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v22.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v27.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v31.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_3 @@ -6334,188 +6333,194 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v19.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v16.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v15.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v0.h, v15.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v12.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v1.h, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v2.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v11.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v3.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v12, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v4.l, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v5.l, v9.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v9.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v6.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v7.l, v8.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v9, v19 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2 ; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v21.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v14.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v19.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v18.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v15.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v15.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v15.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v17.h, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v19.l, 3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v13.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v16.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v15.h, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v14.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v13.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v18.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v10.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v11.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.h, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v12.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v12.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v18.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.h, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v21 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v28.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v3.h ; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v10.h, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v11.l, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v8.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v8.h, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v9.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v9.h, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v9.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v8.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v21 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v8.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v8, v21 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -13330,33 +13335,32 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v19.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v17.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v13.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v22.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v22.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v27.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v31.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3 @@ -13368,188 +13372,194 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v19.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v16.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v15.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v0.h, v15.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v12.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v1.h, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v2.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v11.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v3.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v12, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v4.l, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v5.l, v9.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v9.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v6.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v7.l, v8.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v9, v19 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2 ; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v21.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v14.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v19.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v18.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v15.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v15.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v15.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v17.h, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v19.l, 3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v13.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v16.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v15.h, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v14.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v13.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v18.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v10.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v11.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.h, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v12.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v12.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v18.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.h, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v21 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v28.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v3.h ; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v10.h, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v11.l, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v8.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v8.h, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v9.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v9.h, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v9.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v8.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v21 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v8.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v8, v21 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -19882,33 +19892,32 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v19.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v17.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v13.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v22.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v22.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v27.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v31.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_3 @@ -19920,188 +19929,194 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v19.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v16.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v15.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v0.h, v15.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v12.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v1.h, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v2.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v11.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v3.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v12, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v4.l, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v5.l, v9.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v9.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v6.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v7.l, v8.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v9, v19 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2 ; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v21.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v14.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v19.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v18.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v15.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v15.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v15.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v17.h, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v19.l, 3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v13.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v16.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v15.h, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v14.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v13.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v18.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v10.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v11.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.h, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v12.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v12.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v18.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.h, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v21 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v28.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v3.h ; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v10.h, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v11.l, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v8.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v8.h, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v9.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v9.h, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v9.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v8.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v21 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v8.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v8, v21 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -25924,33 +25939,32 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v19.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v17.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v13.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v25.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v24.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v22.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v21.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v22.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v21.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v27.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v31.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_3 @@ -25962,188 +25976,194 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v19.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v16.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v15.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v12.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.h, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v4.h, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v0.h, v15.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v12.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v1.h, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v2.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v11.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v3.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v12, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v4.l, v10.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v5.l, v9.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v9.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v6.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v7.l, v8.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v9, v19 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2 ; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v21.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v14.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v19.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v18.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v17.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v15.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v18.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v15.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v15.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v19.l, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v17.h, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v19.l, 3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v13.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v16.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v13.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v15.h, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v14.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v13.l, v1.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v14.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v18.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v10.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v11.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.h, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v12.l, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v21 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v12.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v18.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v11.h, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v21 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v28.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v3.h ; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v10.h, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v11.l, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v8.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v8.h, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v9.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v9.h, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v3.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v4.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v9.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v15, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v6, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v8.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v21 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v8.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v21.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v8, v21 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll index e773b54..0cefbc1 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll @@ -2966,20 +2966,20 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 @@ -2995,17 +2995,17 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB12_2: ; %Flow @@ -3029,17 +3029,17 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB12_4: ; %end @@ -3047,100 +3047,105 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v2.h, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v1.h, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v28, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v15 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v4.l, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v15, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v6.h, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v29, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v30, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v26, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v13, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v19, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v17.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v15 ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[9:10], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[10:11], off offset:32 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v10i32_to_v40i8: @@ -5033,50 +5038,48 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:24 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:20 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v25.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v23.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v21.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v15.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v27.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v29.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v29.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v33.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v33.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v34.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v35.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v36 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_3 @@ -5091,228 +5094,243 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v26.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v25.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v21.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v24.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v24.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v17 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v17, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v0.h, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v19.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v1.h, v19.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v2.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v3.l, v15.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v4.l, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v14.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v25 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v5.l, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v6.h, v13.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v6.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v7.l, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v25 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v8.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v25 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v9.l, v10.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v25 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2 ; GFX11-TRUE16-NEXT: .LBB14_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v26.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v25.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v25.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v21.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v22.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v21.h, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v18.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v24.h, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v23.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v22.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v23.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v24.h, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v15.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v19.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v16.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v20.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v15.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v17.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v18.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.h, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v18.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v21.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v19.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v16.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v17.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v27 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v19.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v16.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v27 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v15.h, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v17.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v18.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v14.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v15.l, v4.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v4.l ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v14.l, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v13.h, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.h, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v13.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v14.h, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v13.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v14.l, v6.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v13.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v12.h, v6.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v27 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v10.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v11.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v11.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v27 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v12.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v11.l, v7.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v17, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v11.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v10.h, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v27 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v10, v27 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -9933,20 +9951,20 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 @@ -9962,17 +9980,17 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB32_2: ; %Flow @@ -9992,17 +10010,17 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[5:6] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB32_4: ; %end @@ -10010,100 +10028,105 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v2.h, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v1.h, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v28, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v15 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v4.l, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v15, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v6.h, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v29, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v30, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v26, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v13, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v19, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v17.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v15 ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[9:10], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[10:11], off offset:32 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v10f32_to_v40i8: @@ -12014,50 +12037,48 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:24 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:32 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:20 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v25.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v23.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v21.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v15.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v28.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v27.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v30.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v29.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v27.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v29.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v33.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v33.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v33.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v34.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v35.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v35.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v36 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_3 @@ -12072,228 +12093,243 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v26.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v25.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v25.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v20.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v21.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v24.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v24.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v19.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v17 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v17, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v23.h +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v0.h, v21.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v19.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v1.h, v19.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v16.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v2.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v3.l, v15.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v4.l, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v14.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v25 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v5.l, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v6.h, v13.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v6.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v25 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v7.l, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v25 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v8.l, v10.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v25 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v9.l, v10.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v25 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_2 ; GFX11-TRUE16-NEXT: .LBB34_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v26.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v25.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v25.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v21.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v22.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v21.h, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v18.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v24.h, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v23.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v22.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v23.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v24.h, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v15.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v19.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v16.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v20.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v15.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v17.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v18.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v16.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v17.h, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v21.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v18.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v21.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v19.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v16.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v17.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v27 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v19.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v16.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v27 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v15.h, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v17.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v18.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v14.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v15.l, v4.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v4.l ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v27 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v14.l, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v13.h, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.h, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v13.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v14.h, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v13.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v14.l, v6.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v17 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v13.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v12.h, v6.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v27 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v10.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v11.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v11.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v27 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v12.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v11.l, v7.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v17, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v11.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v10.h, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v27 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v10.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v27.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v10, v27 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -16322,20 +16358,20 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 @@ -16351,17 +16387,17 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB48_2: ; %Flow @@ -16385,17 +16421,17 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB48_4: ; %end @@ -16403,100 +16439,105 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v2.h, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v1.h, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v28, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v15 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v4.l, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v15, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v6.h, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v29, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v30, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v26, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v13, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v19, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v17.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v15 ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[9:10], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[10:11], off offset:32 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v20i16_to_v40i8: @@ -22438,20 +22479,20 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 @@ -22467,17 +22508,17 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB60_2: ; %Flow @@ -22501,17 +22542,17 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB60_4: ; %end @@ -22519,100 +22560,105 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v2.h, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v1.h, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v28, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v15 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v4.l, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v15, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v6.h, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v29, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v30, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v26, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v13, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v19, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v17.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v15 ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[9:10], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[10:11], off offset:32 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v20f16_to_v40i8: @@ -28813,50 +28859,50 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:24 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:32 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:20 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v27.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v25.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v38.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v38.h ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v36.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v36.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v36.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v37.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v38.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v49 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB72_3 @@ -28871,228 +28917,243 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v35.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v34.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v34.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v30.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v0.h, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v24.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v25.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v27.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v1.h, v27.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v2.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v22.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v3.l, v23.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v4.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v21.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v5.l, v19.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v6.h, v19.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v6.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v7.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v11, v10 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v8.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v9.l, v16.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v10 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_2 ; GFX11-TRUE16-NEXT: .LBB72_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v34.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v29.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v29.h, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v34.l, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v33.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v30.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v34.l, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v27.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v23.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v27.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v22.h, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v28.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v23.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v25.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v21.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v24.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v25.h, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v29.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v26.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v29.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v27.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v25.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v27.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v23.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v23.h, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v25.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v21.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v22.h, v4.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v4.l ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v21.l, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v19.h, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v18.h, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v19.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v21.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v19.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v20.h, v6.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v19.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v18.h, v6.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v11 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v16.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v16.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v17.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v17.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v18.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v17.l, v7.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v17 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v17.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v16.h, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v16.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v10, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -30847,20 +30908,20 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 @@ -30876,17 +30937,17 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB74_2: ; %Flow @@ -30905,17 +30966,17 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB74_4: ; %end @@ -30923,100 +30984,105 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v2.h, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v1.h, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v28, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v15 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v4.l, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v15, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v6.h, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v29, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v30, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v26, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v13, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v19, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v17.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v15 ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[9:10], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[10:11], off offset:32 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v5f64_to_v40i8: @@ -32944,50 +33010,50 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:24 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:32 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:20 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v27.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v25.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v23.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v6.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v38.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v39.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v39.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v38.h ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(8) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v36.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v36.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v36.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v37.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(4) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v38.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v38.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v49 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_3 @@ -33002,228 +33068,243 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v35.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v34.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v29.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v34.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v33.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v30.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v33.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v34.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v0.h, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v22.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v24.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v25.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v5.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v27.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v26.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v1.h, v27.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v2.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v20.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v22.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v3.l, v23.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v4.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v21.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v5.l, v19.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v6.h, v19.l +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v6.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v7.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v31.h +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v11, v10 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v8.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v9.l, v16.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v10 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_2 ; GFX11-TRUE16-NEXT: .LBB76_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v34.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v29.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v30.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v29.h, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v34.l, v0.l ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v33.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v30.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v33.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v34.l, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v27.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v23.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v27.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v22.h, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v28.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v23.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v25.l, v1.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v21.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v24.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v25.h, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v29.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v20.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v26.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v29.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v27.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v25.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v27.h, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v23.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v23.h, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v25.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v20.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v20.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v21.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v22.h, v4.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v22.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v4.l ; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h ; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v21.l, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v19.h, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v18.h, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v19.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v21.l, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v19.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v20.h, v6.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v30.l, 3 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v5.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v19.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v18.h, v6.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v11 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v32.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v16.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v16.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v17.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v17.h, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v18.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v3, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v5.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v5.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v17.l, v7.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v14, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v17 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v17.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v16.h, v8.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v16.l, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, 0x300, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v10, v11 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -34993,20 +35074,20 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 @@ -35022,17 +35103,17 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB78_2: ; %Flow @@ -35059,17 +35140,17 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[15:16], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v1 ; GFX11-TRUE16-NEXT: .LBB78_4: ; %end @@ -35077,100 +35158,105 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v16.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v30.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v13.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v2.h, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v1.h, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v28.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff, v29 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v11.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v28, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v3.l, v3.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v15 ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v4.l, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v15, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v22.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v13.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v6.h, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v29, v16 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v14, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v13, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v7.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v12, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v11.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v30, v28 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v26, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v15, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v13, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v19, v10 +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v13 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v17.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v11, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v11, v15 ; GFX11-TRUE16-NEXT: s_clause 0x2 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 -; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[9:10], off offset:32 +; GFX11-TRUE16-NEXT: scratch_store_b64 v0, v[10:11], off offset:32 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: bitcast_v5i64_to_v40i8: diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll index bfed1f4..48c9b87 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll @@ -2257,8 +2257,8 @@ define i32 @bitcast_v4i8_to_i32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4 @@ -2273,18 +2273,19 @@ define i32 @bitcast_v4i8_to_i32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: .LBB22_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2 ; GFX11-TRUE16-NEXT: .LBB22_4: ; %cmp.true @@ -2294,16 +2295,17 @@ define i32 @bitcast_v4i8_to_i32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4504,8 +4506,8 @@ define float @bitcast_v4i8_to_f32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4 @@ -4520,18 +4522,19 @@ define float @bitcast_v4i8_to_f32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: .LBB42_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_2 ; GFX11-TRUE16-NEXT: .LBB42_4: ; %cmp.true @@ -4541,16 +4544,17 @@ define float @bitcast_v4i8_to_f32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -6463,8 +6467,8 @@ define <2 x i16> @bitcast_v4i8_to_v2i16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4 @@ -6479,18 +6483,19 @@ define <2 x i16> @bitcast_v4i8_to_v2i16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: .LBB58_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB58_2 ; GFX11-TRUE16-NEXT: .LBB58_4: ; %cmp.true @@ -6500,16 +6505,17 @@ define <2 x i16> @bitcast_v4i8_to_v2i16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -8110,8 +8116,8 @@ define <2 x half> @bitcast_v4i8_to_v2f16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4 @@ -8126,18 +8132,19 @@ define <2 x half> @bitcast_v4i8_to_v2f16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2 ; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true @@ -8147,16 +8154,17 @@ define <2 x half> @bitcast_v4i8_to_v2f16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -9471,8 +9479,8 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4 @@ -9487,18 +9495,19 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: .LBB78_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB78_2 ; GFX11-TRUE16-NEXT: .LBB78_4: ; %cmp.true @@ -9508,16 +9517,17 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -10183,8 +10193,8 @@ define <1 x i32> @bitcast_v4i8_to_v1i32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4 @@ -10199,18 +10209,19 @@ define <1 x i32> @bitcast_v4i8_to_v1i32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: .LBB82_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2 ; GFX11-TRUE16-NEXT: .LBB82_4: ; %cmp.true @@ -10220,16 +10231,17 @@ define <1 x i32> @bitcast_v4i8_to_v1i32(<4 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll index 45e205b..68312b8 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll @@ -145,37 +145,36 @@ define <3 x half> @bitcast_v3bf16_to_v3f16(<3 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x7fc0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h ; GFX11-TRUE16-NEXT: .LBB0_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -797,40 +796,40 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, v2 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, 0x7fc0, 16, v1 +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1 +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, 0x7fc0, 16, v2 ; GFX11-TRUE16-NEXT: .LBB4_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll index d524514..5aac06a 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll @@ -8768,32 +8768,32 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 @@ -8812,26 +8812,26 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB24_2: ; %Flow @@ -8864,26 +8864,26 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB24_4: ; %end @@ -8891,151 +8891,156 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v24.l ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.h, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v2.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v21.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v25, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.l, v3.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v25, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v7.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v19.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v53, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v8.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v21, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v36.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v21, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v20, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v24 ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v34 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v12.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v17.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v28.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v25, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v49, v50 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v35, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v22, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 @@ -12465,13 +12470,13 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:124 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:120 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:116 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:112 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:88 ; GFX11-TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:132 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 @@ -12487,81 +12492,83 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:80 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:84 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v8.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v80.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v80.h ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v64.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v65.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v66.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(21) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v67.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v67.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v67.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v68.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v69.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v69.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v70.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(14) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v70.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v71.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v71.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v80.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v82 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_3 @@ -12574,366 +12581,384 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v55.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v54.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v53.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v53.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v50.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v50.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v51.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v52.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v2.h, v51.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v6, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v0.h, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v52.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v50.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v1.h, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v2.l, v49.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v3.l, v48.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v25.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v24.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v25.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v26.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v20.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v4.l, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v29.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v5.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v6.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v7.l, v23.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v8.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v35.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v9.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v21.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v10.l, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v11.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v12.h, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v12.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v13.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v14.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v54 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v15.l, v16.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v54 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2 ; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v55.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v54.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v53.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v52.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v53.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v50.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v51.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v52.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v51.l, v0.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v50.h, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v29.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v29.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v27.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v27.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v30.h, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v39.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v50.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v30.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v49.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v51.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v39.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v48.l, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v48.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v48.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v27.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v49.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v27.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v39.l, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v29.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v25.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v29.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v28.h, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v23.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v25.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v24.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v25.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v26.h, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v37.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v26.h, v6.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v25.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v21.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v22.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v36.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v21.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v22.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v24.l, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v23.h, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v22.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v23.l, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v34.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v21.h, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v10, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v22.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v20.h, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v21.l, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h ; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v20.l, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.h, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v18.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v19.h, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v20.h, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v20.l, v12.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v11.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v19.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v18.h, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v32.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v16.l, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v16.h, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v17.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v17.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h ; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v18.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v17.l, v13.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v17.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v16.h, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v16, v55 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -23563,32 +23588,32 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 @@ -23607,26 +23632,26 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB48_2: ; %Flow @@ -23651,26 +23676,26 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB48_4: ; %end @@ -23678,151 +23703,156 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v24.l ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.h, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v2.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v21.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v25, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.l, v3.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v25, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v7.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v19.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v53, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v8.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v21, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v36.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v21, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v20, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v24 ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v34 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v12.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v17.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v28.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v25, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v49, v50 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v35, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v22, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 @@ -27383,13 +27413,13 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:124 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:120 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:116 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:112 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:88 ; GFX11-TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:132 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 @@ -27405,81 +27435,83 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:80 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:84 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v8.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v80.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v80.h ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v64.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v65.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v66.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(21) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v67.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v67.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v67.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v68.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v69.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v69.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v70.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(14) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v70.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v71.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v71.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v80.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v82 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3 @@ -27492,366 +27524,384 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v55.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v54.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v53.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v53.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v50.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v50.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v51.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v52.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v2.h, v51.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v6, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v0.h, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v52.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v50.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v1.h, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v2.l, v49.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v3.l, v48.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v25.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v24.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v25.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v26.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v20.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v4.l, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v29.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v5.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v6.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v7.l, v23.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v8.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v35.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v9.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v21.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v10.l, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v11.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v12.h, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v12.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v13.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v14.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v54 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v15.l, v16.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v54 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2 ; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v55.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v54.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v53.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v52.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v53.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v50.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v51.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v52.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v51.l, v0.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v50.h, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v29.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v29.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v27.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v27.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v30.h, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v39.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v50.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v30.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v49.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v51.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v39.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v48.l, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v48.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v48.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v27.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v49.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v27.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v39.l, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v29.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v25.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v29.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v28.h, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v23.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v25.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v24.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v25.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v26.h, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v37.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v26.h, v6.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v25.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v21.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v22.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v36.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v21.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v22.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v24.l, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v23.h, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v22.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v23.l, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v34.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v21.h, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v10, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v22.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v20.h, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v21.l, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h ; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v20.l, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.h, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v18.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v19.h, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v20.h, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v20.l, v12.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v11.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v19.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v18.h, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v32.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v16.l, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v16.h, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v17.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v17.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h ; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v18.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v17.l, v13.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v17.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v16.h, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v16, v55 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -37866,32 +37916,32 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 @@ -37910,26 +37960,26 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB68_2: ; %Flow @@ -37967,26 +38017,26 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB68_4: ; %end @@ -37994,151 +38044,156 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v24.l ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.h, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v2.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v21.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v25, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.l, v3.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v25, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v7.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v19.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v53, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v8.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v21, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v36.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v21, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v20, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v24 ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v34 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v12.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v17.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v28.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v25, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v49, v50 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v35, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v22, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 @@ -41573,13 +41628,13 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:124 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:120 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:116 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:112 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:88 ; GFX11-TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:132 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 @@ -41595,81 +41650,83 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:80 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:84 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v8.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v80.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v80.h ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v64.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v65.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v66.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(21) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v67.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v67.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v67.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v68.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v69.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v69.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v70.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(14) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v70.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v71.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v71.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v80.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v82 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_3 @@ -41682,366 +41739,384 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v55.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v54.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v53.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v53.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v50.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v50.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v51.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v52.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v2.h, v51.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v6, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v0.h, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v52.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v50.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v1.h, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v2.l, v49.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v3.l, v48.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v25.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v24.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v25.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v26.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v20.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v4.l, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v29.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v5.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v6.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v7.l, v23.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v8.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v35.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v9.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v21.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v10.l, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v11.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v12.h, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v12.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v13.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v14.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v54 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v15.l, v16.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v54 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2 ; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v55.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v54.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v53.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v52.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v53.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v50.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v51.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v52.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v51.l, v0.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v50.h, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v29.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v29.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v27.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v27.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v30.h, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v39.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v50.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v30.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v49.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v51.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v39.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v48.l, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v48.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v48.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v27.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v49.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v27.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v39.l, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v29.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v25.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v29.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v28.h, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v23.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v25.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v24.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v25.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v26.h, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v37.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v26.h, v6.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v25.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v21.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v22.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v36.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v21.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v22.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v24.l, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v23.h, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v22.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v23.l, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v34.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v21.h, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v10, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v22.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v20.h, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v21.l, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h ; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v20.l, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.h, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v18.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v19.h, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v20.h, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v20.l, v12.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v11.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v19.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v18.h, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v32.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v16.l, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v16.h, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v17.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v17.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h ; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v18.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v17.l, v13.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v17.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v16.h, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v16, v55 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -51220,32 +51295,32 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 @@ -51264,26 +51339,26 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB84_2: ; %Flow @@ -51308,26 +51383,26 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB84_4: ; %end @@ -51335,151 +51410,156 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v24.l ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.h, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v2.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v21.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v25, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.l, v3.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v25, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v7.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v19.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v53, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v8.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v21, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v36.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v21, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v20, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v24 ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v34 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v12.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v17.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v28.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v25, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v49, v50 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v35, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v22, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 @@ -54909,13 +54989,13 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:128 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:124 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v64, off, s32 offset:120 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:116 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:116 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:112 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:108 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:108 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v65, off, s32 offset:104 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:100 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:100 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:96 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:92 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:92 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v66, off, s32 offset:88 ; GFX11-TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:132 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 @@ -54931,81 +55011,83 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v80, off, s32 offset:80 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:84 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:76 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:68 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:60 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:68 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:60 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v35, off, s32 offset:52 ; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:44 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:36 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:28 -; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:20 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:36 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v36, off, s32 offset:28 +; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:20 ; GFX11-TRUE16-NEXT: s_clause 0x1 -; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:12 +; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v37, off, s32 offset:12 ; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v38, off, s32 offset:4 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v29.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v20.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v18.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v24.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v20.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v8.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v81.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 8, v80.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v54.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v53.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v50.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v52.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v51.l, 8, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v48.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v49.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v39.l, 8, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 8, v25.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v81.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v80.h ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(33) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v64.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(31) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(29) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v65.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(27) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v65.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v65.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(25) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v66.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v66.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(23) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v66.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v66.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(21) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v67.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(20) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v67.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.l, 8, v67.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(19) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v68.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v68.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(18) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v68.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(17) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v69.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(16) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v69.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 8, v69.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(15) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 8, v70.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v70.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(14) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v70.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v70.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(13) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v71.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(12) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v71.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v71.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(11) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.l, 8, v80.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v80.l ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v82 ; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_3 @@ -55018,366 +55100,384 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v55.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v54.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v53.h +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v51.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v53.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v53.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v49.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v54.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v54.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v50.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v49.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v50.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v52.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v51.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v52.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v2.h, v51.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v6, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v28.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v27.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v27.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v39.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.h, v48.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v0.h, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v52.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v50.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v51.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.l, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v1.h, v50.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v27.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v7, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v2.l, v49.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v3.l, v48.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v24.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v23.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v25.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v24.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v25.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v26.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v38.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v37.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v37.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v36.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v6.h, v21.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v7.h, v22.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v35.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v20.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v33.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v31.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v11.h, v16.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v12.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v18.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v38.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v4.l, v29.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v29.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v5.l, v28.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v5.h, v26.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v38.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v6.l, v25.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.h, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v37.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v7.l, v23.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v37.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v7.h, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v8.l, v22.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v35.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v11, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v9.l, v21.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v36.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v9.h, v21.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v10.l, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v10.h, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v11.l, v19.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v34.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v12.h, v19.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v33.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v12.l, v18.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v12.h, v18.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v54 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v13.l, v17.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v14.l, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v31.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v17, v54 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v54.h, v15.l, v16.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v54 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2 ; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v55.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v54.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v51.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v53.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v49.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v54.h, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v52.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v48.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v54.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v53.h, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v50.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v52.l, v0.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v51.h, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v52.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v51.l, v0.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v50.h, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v29.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v29.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v28.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v27.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v27.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v30.h, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v39.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v52.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v50.h, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v39.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v30.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v49.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v51.l, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h ; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v39.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v48.l, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v24.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v28.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v48.h, v3.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v26.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v30.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v48.l, v3.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v27.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v49.l, v3.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v27.h, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v39.l, v4.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v29.h, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v25.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v4.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v29.l, v5.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v28.l, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v28.h, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v26.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v23.h, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v25.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v9 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v24.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v25.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v10, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v38.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v37.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v26.h, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, v37.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v36.h, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v5.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v26.h, v6.h +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v8, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v25.h, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, v30.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v21.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v22.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v36.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v21.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v22.h, v7.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v6.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v6.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v24.l, v7.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v37.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v23.h, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, v38.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v7.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v6.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, 0x300, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v11, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v13, v14 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v22.h, v8.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v36.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l ; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v23.l, v8.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v35.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v34.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v21.h, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v10, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v22.l, v9.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.l, v36.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v35.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v9.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v11, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v20.h, v10.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v35.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v21.l, v10.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v34.h, 3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v10.h ; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v33.h, 3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v13, v55 +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v20.l, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v34.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.h, v11.h ; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v18.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v19.h, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v20.h, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8 -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v19.l, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v20.l, v12.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v15, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v23 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, v33.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.h, v32.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, v32.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.h, v31.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v31.l, 3 -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v11.h +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v19.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v18.h, v12.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v13, v55 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, v33.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v12.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.h, v32.h, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v16.l, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v16.h, v11.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v17.l, v12.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v17.h, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.h, v32.l, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v15, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h ; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v18.l, v13.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v11.l, 0x300, v11.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, 0x300, v11.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v12.l, 0x300, v12.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, 0x300, v12.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v14.l, v31.h, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v15 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v17.l, v13.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v13.l, 0x300, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v23, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v14, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v15, v18 +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v17.h, v14.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v15.l, v31.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v13.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v16.h, v14.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v17.l, 0x300, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v16.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v17 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v55.h, 0x300, v15.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v16, v55 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -64473,32 +64573,32 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 @@ -64517,26 +64617,26 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB96_2: ; %Flow @@ -64569,26 +64669,26 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB96_4: ; %end @@ -64596,151 +64696,156 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v24.l ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.h, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v2.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v21.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v25, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.l, v3.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v25, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v7.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v19.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v53, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v8.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v21, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v36.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v21, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v20, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v24 ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v34 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v12.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v17.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v28.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v25, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v49, v50 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v35, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v22, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 @@ -76596,32 +76701,32 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16 @@ -76640,26 +76745,26 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB104_2: ; %Flow @@ -76692,26 +76797,26 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 24, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 24, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 8, v11 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v1 ; GFX11-TRUE16-NEXT: .LBB104_4: ; %end @@ -76719,151 +76824,156 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v24.l ; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v64.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h ; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v17.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v64.l -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.h, v18.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v54.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v18.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v53.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v2.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v19.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v21.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v24 -; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.h, v20.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v54.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v53.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v55 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v25, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v2.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v54, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v52.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.l, v3.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v4.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v51.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v25 ; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v25 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 8, v37.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v50.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v49.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v23, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v54, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v25, v22 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_or_b16 v22.l, v7.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v38.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v19.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v53, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v23.l, v8.h, v20.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v21 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v36.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v34.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v48.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v39.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v22, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v23 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v21, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v21, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v17.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v37.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v36.l ; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.l, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v12.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v32.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 8, v18.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v22 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v21, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v35.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v34.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v20, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v33.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v24 ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v19.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v34 -; GFX11-TRUE16-NEXT: v_or_b16 v20.l, v12.h, v18.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v13.h, v18.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v32.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v21 +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v20 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v17.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v18, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v17.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v14.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v28.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v14.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v28.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v14.h +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v16.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v26.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v25, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v49, v50 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v51, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v35, v11 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v21, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v22, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v19, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 @@ -85582,58 +85692,58 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16 ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -85648,297 +85758,304 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[5:6] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[23:24], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 24, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 24, v12 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 8, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 8, v12 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v10 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v10 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v9 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 24, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v7 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v6 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v5 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v2 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v1 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v2.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v3.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v4.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v4.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v6.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v8.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v8.h ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.h, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v10.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v10.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.h, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.h, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v12.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v13.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.h, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.h, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.h, v12.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v13.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v14.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v16.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v16.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v14.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v15.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v16.h ; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v17, 0x40c00000, v17 -; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v17, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v17 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 -; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v2 ; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v17, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v20, v22, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v27.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v20, v22, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18 -; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v26.h -; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v21, v23, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v21, v23, vcc_lo ; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18 +; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v19 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v27 +; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v20, v21, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18 ; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v4, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 24, v2 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v28.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v17, v22, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v4, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v20, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v2 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v18, v22, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_lshlrev_b32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v17, v23, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v30.h +; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 ; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v21, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v29, v17, v23 ; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v3, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v21, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21 ; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v31, v18, v19 :: v_dual_add_f32 v18, 0x40c00000, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v30.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v18, v19, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v31.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v23, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v20, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v32.h +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v4, v23 :: v_dual_add_f32 v18, 0x40c00000, v22 ; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v17, v29 ; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v6, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v19, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v18, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18 ; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v6, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v20, 16, 1 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v4 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v4 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v17, v21, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v18, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 8, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v17, v21, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v20, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v19, v22, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v31, v19, v22 :: v_dual_and_b32 v20, 0xffff0000, v5 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_lshlrev_b32 v5, 16, v5 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v8, 16, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v20, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v32.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v33.h ; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v5, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v20, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v17, v21, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v17, v21, vcc_lo ; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v8, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v35.h +; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v36.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v6, v22, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v19, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v6, v17, v8, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v8 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v34, v6, v17 :: v_dual_add_f32 v19, 0x40c00000, v19 -; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v18, v33 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v19, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 24, v6 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v6 ; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v19, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v36, v5, v22 :: v_dual_and_b32 v23, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v6, v17, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v18, v31 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v5, v22, vcc_lo ; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v21, v20 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v36 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v35.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 24, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v7, 0x40c00000, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 24, v8 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v8 +; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v34 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 24, v8 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v8 ; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 ; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v20, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v19, v21, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v19, v21, vcc_lo ; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v23 :: v_dual_add_f32 v10, 0x40c00000, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v20, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v17 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 ; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v10, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v10, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v22, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v10 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v49.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v7, v21, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v39, v7, v21, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v12 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v48, v19, v22 :: v_dual_lshlrev_b32 v7, 16, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v38.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_add_f32 v7, 0x40c00000, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v10 -; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v19, v48 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v12, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v7, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v39.h +; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v21, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v7 -; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v19, v48 +; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v12, 16, 1 ; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v21, 0x7fff -; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v23 -; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v12, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v12 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v39 :: v_dual_cndmask_b32 v52, v22, v37 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 24, v10 +; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v12, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v22, v37, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v7, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v7 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v9, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v9 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v52.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 +; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v7, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v54.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_lshlrev_b32 v11, 16, v11 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v65, v19, v25, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v9, 0x7fff -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v9 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v38 +; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v52, v24, v50 :: v_dual_add_f32 v9, 0x40c00000, v23 ; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v21, 16, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v24, v50, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v7, v52 +; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v9, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v9 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v10 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v10 -; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v7, v53 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v20, v23, vcc_lo ; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v11 ; GFX11-TRUE16-NEXT: v_add3_u32 v11, v19, v21, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v9, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v21 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 24, v12 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 8, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v20, v23, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21 -; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1 ; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v7, 16, 1 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v65.h -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 24, v12 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v19, vcc_lo ; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add3_u32 v21, v23, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v7 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v13 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v19, 16, 1 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v20, v9 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v23 :: v_dual_cndmask_b32 v70, v21, v22 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v23 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v71, v21, v22, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v21, v24, v19, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19 ; GFX11-TRUE16-NEXT: v_add3_u32 v23, v25, v14, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v14 ; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v7, 16, 1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v67, v21, v22, vcc_lo +; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v13 :: v_dual_cndmask_b32 v66, v21, v22 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add3_u32 v14, v25, v7, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v68, v23, v24, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v13, 16, 1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 8, v12 +; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v20, v9 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v68.h ; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v14, v19 :: v_dual_add_f32 v14, 0x40c00000, v21 @@ -85950,42 +86067,42 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v21 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v16 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v19, v23, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v86, v19, v23, vcc_lo ; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v16, 0x7fff ; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v21, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 ; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v14, 16, 1 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v21 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v21 ; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v21, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v13, v25, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v13, v25, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21 ; GFX11-TRUE16-NEXT: v_add3_u32 v19, v24, v14, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v14 ; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v15, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v84, v23, v39, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v23, v38, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v85.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v86.h ; GFX11-TRUE16-NEXT: v_add3_u32 v13, v37, v15, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v70.h -; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v22, v67 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v19, v24, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v71.h +; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v22, v66 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v19, v24, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v81.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v84.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v82.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v85.h ; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v23, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 24, v14 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 24, v14 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v19, v82 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v14 +; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v19, v81 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 8, v14 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 8, v11 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 8, v9 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v9 ; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v13 ; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v21, v7 ; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v18, v17 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v16 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 8, v16 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 8, v16 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[15:16] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[13:14] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[11:12] @@ -85994,160 +86111,159 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[5:6] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[23:24], 24, v[3:4] ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2] -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v15 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v55, 8, v13 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v7 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 8, v15 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v13 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 8, v7 ; GFX11-TRUE16-NEXT: .LBB108_4: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v28.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v113.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v24.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v27.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v112.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v27.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v26.h ; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v31.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.h, v2.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v26.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v102.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v103.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v24.l ; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v2.h, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v101.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v23.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v8 -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v35.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v99.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v30.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v29.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v100.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v3.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v22.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v14, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v49.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v103.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v32.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v1.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v102.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v30.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v101.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v24 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v23.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v2.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v36.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v3.h, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v29.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v99.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v100.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v8, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v98.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.h, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v4.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v33.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v97.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h +; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v49.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v8, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v31.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v96.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v10, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v6.h, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v10, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v12 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v34.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v87.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v36.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v86.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v65.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v83.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v16, v22 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.h, v11.l -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v20.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v32.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v98.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v38.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v80.l -; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v48.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v71.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v7.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.l, v5.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v6.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v35.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v87.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v10, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v7.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v65.h +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v7.h, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v84.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v69.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v10, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v83.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v34.h +; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v8.h, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v39.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v80.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v8.l, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.h, v10.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v24 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v71.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v9.l, v9.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v48.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v70.l +; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v10.h, v11.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v14, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v67.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v10.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v54.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l ; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v11.h ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v8.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v9.l, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v23, v24 -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v70.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v69.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v16, v21 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v12, v22 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v14, v19 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v64.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v12.h, 0xff, v86.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v14, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v16 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v11.l, v11.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v12.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v52.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v66.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v53.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v64.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v55.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v12.h, v13.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v16, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v12.l, v14.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v13.h +; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v68.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v53.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v16, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff, v19 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v13.l, v14.h ; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v85.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v55.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v18.l -; GFX11-TRUE16-NEXT: v_and_b16 v16.h, 0xff, v68.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v54.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v33.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v97.l -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v10.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.h, v15.l -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v13.h, v16.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.h, v17.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v11.l, v11.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v20, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v67.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v51.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v84.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v50.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v15.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v81.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v39.l -; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v82.h +; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v13.h, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.l, 8, v51.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v16, v24 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v50.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v14.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v66.h +; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v14.h, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, 0xff, v82.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v38.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff, v18 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v14.l, v16.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v15.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v14.h, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v18, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v19 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v37.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v13.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v14.l, v14.l, v14.h -; GFX11-TRUE16-NEXT: v_or_b16 v15.l, v15.l, v15.h -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v16.h -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v25, v6 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v24, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v19, v20 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v21, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v22, v17 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v23, v24 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v25, v16 +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v15.l, v15.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v81.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v17, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v16.l, v16.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v17, v24 ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off ; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16 diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll index a40ee16..6fe6665 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll @@ -2160,46 +2160,47 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v7, v10 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4 ; GFX11-TRUE16-NEXT: .LBB22_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -3064,12 +3065,13 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -3083,66 +3085,61 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2 ; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -5307,46 +5304,47 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v7, v10 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4 ; GFX11-TRUE16-NEXT: .LBB46_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -6216,12 +6214,13 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -6235,66 +6234,61 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2 ; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -8166,46 +8160,47 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v7, v10 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4 ; GFX11-TRUE16-NEXT: .LBB66_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -9068,12 +9063,13 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -9087,66 +9083,61 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2 ; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -10698,46 +10689,47 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v7, v10 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 ; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4 ; GFX11-TRUE16-NEXT: .LBB82_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -11611,12 +11603,13 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -11630,66 +11623,61 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2 ; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -12867,49 +12855,49 @@ define <4 x i16> @bitcast_v4bf16_to_v4i16(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4 ; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v0, 16, v1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v3, 16, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v2, 16, v3 ; GFX11-TRUE16-NEXT: .LBB94_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -13841,12 +13829,13 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -13860,66 +13849,61 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2 ; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -14696,46 +14680,46 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_add_f32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h ; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0 ; GFX11-TRUE16-NEXT: .LBB102_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -15671,12 +15655,13 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -15690,66 +15675,61 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2 ; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -16347,42 +16327,41 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v8.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v0 :: v_dual_lshlrev_b32 v0, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v12, v4, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v10, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v0 :: v_dual_add_f32 v0, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l +; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v11, v4, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v12, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v11, v5, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v3, v2 -; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v1, v6 +; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v2, v1 +; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v3, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[10:11], 24, v[8:9] @@ -16987,12 +16966,13 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v7.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8 @@ -17006,66 +16986,61 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v4 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2 ; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v5.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll index 5163539..e5245f7 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll @@ -1102,15 +1102,15 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v0.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v11.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12 @@ -1126,76 +1126,79 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v0.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v3.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v5 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v2.l, v3.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v7 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_2 ; GFX11-TRUE16-NEXT: .LBB6_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v7.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v6.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v5.h, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v4.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v4, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v7 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -2084,62 +2087,57 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v5, v7, v9 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v11, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v3, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v11 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v6 :: v_dual_cndmask_b32 v0, v0, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v11, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v10, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v5, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5 +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v3, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v6, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v5 ; GFX11-TRUE16-NEXT: .LBB10_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -4243,15 +4241,15 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v0.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v11.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12 @@ -4267,76 +4265,79 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v7.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v7.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v5.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v0.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v4.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v3.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v5 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v2.l, v3.l +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v7 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2 ; GFX11-TRUE16-NEXT: .LBB22_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v7.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v7.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v6.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v6.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v5.h, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v4.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v0.h +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v4, v7 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v7 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -5229,62 +5230,57 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v5, v7, v9 +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v11, 16, 1 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v3, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v12, vcc_lo -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v11 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v6 :: v_dual_cndmask_b32 v0, v0, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v11, 0x7fff +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v10, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v5, 0x7fff ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5 +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v3, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v6, v1 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v5 ; GFX11-TRUE16-NEXT: .LBB26_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -6889,16 +6885,16 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v11.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12 @@ -6914,77 +6910,79 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v1.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v2.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v7 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB36_2 ; GFX11-TRUE16-NEXT: .LBB36_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v7.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v6.h, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v5.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.l, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.h, v2.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v9 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -7783,67 +7781,68 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4 ; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v13 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v13.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v12.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v10 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v11, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v11, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v10, v4, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v13 -; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff ; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v4, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h +; GFX11-TRUE16-NEXT: v_add3_u32 v13, v0, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v12 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v12, v7, 0x7fff -; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v1, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v13, v14, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v5, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v12, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v7, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v2, v1 ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0x7fc07fc0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[12:13] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v7, v3 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v5, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13 ; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11] ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10 ; GFX11-TRUE16-NEXT: .LBB38_4: ; %end @@ -8652,16 +8651,16 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v11.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12 @@ -8677,77 +8676,79 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v1.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v2.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v7 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB40_2 ; GFX11-TRUE16-NEXT: .LBB40_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v7.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v6.h, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v5.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.l, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.h, v2.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v9 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -10064,16 +10065,16 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v9.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v10.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v9.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v10.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v9.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v11.l ; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 ; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12 @@ -10089,77 +10090,79 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) { ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v8.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v7.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v1.h, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v4.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v8.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v4, v3 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v5.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v10.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v3, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v1.l, v4.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v5 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_hi16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v7 +; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v2.l, v4.l ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16 ; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16 -; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v7 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_hi16 +; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16 ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB44_2 ; GFX11-TRUE16-NEXT: .LBB44_4: ; %cmp.true ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.h, 3 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v7.h, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, 3 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v8.l, 3 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v6.h, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v5.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.l, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v10.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.h, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.h, v2.h ; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x300, v1.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v1.l ; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x300, v1.h -; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v2.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v4.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v9 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v9 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -11446,59 +11449,61 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v6, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff +; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v1, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v1, v8, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v9, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v10, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v3 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v12 :: v_dual_add_f32 v5, 0x40c00000, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v9, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v12 :: v_dual_add_f32 v5, 0x40c00000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v11, v5, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo +; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v6, v0 +; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v7, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v5 ; GFX11-TRUE16-NEXT: .LBB48_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] @@ -12390,64 +12395,66 @@ define <6 x i16> @bitcast_v6bf16_to_v6i16(<6 x bfloat> %a, i32 %b) { ; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB52_2 ; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2 -; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v4, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v4, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v8 :: v_dual_add_f32 v1, 0x40c00000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: v_add3_u32 v11, v12, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v4 +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo ; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9 +; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 -; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v9 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v9, 0x7fff +; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v11, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v7, v10, v1, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v11, v12, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v10 :: v_dual_add_f32 v0, 0x40c00000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v13, vcc_lo +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v13, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h ; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v0, 16, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v1, 16, v4 -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v5, 16, v3 +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v1, 16, v3 +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v5, 16, v4 ; GFX11-TRUE16-NEXT: .LBB52_2: ; %end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll index b7097a9..c7385e4 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -7791,7 +7791,7 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x ; ; GFX6-LABEL: sdiv_i64_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dword s0, s[4:5], 0xd +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) @@ -7927,7 +7927,7 @@ define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x ; ; GFX9-LABEL: sdiv_i64_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s0, s[4:5], 0x34 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34 ; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_lshl_b64 s[0:1], 0x1000, s0 @@ -8982,7 +8982,7 @@ define amdgpu_kernel void @srem_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x ; ; GFX6-LABEL: srem_i64_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dword s0, s[4:5], 0xd +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd ; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) @@ -9116,7 +9116,7 @@ define amdgpu_kernel void @srem_i64_pow2_shl_denom(ptr addrspace(1) %out, i64 %x ; ; GFX9-LABEL: srem_i64_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s0, s[4:5], 0x34 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34 ; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_lshl_b64 s[0:1], 0x1000, s0 @@ -10096,9 +10096,15 @@ define i64 @udiv_i64_9divbits(i8 %size) { } define <2 x i64> @srem_zero_zero() { -; GCN-LABEL: kernel: -; GCN: ; %bb.0: ; %entry -; GCN-NEXT: s_endpgm +; GFX6-LABEL: srem_zero_zero: +; GFX6: ; %bb.0: ; %entry +; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: srem_zero_zero: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] entry: %B = srem <2 x i64> zeroinitializer, zeroinitializer ret <2 x i64> %B diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir new file mode 100644 index 0000000..69bdb1f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir @@ -0,0 +1,95 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX90A %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX908 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefix=NO-AGPR %s + +--- | + define void @func() { + ret void + } + + ; Attribute is ignored for gfx90a + define void @no_agprs() "amdgpu-agpr-alloc"="0,0" { + ret void + } + +... +--- +name: func +tracksRegLiveness: true +stack: + - { id: 0, size: 4 } +body: | + ; HAS-AGPR-LABEL: name: func + ; HAS-AGPR: bb.0: + ; HAS-AGPR-NEXT: successors: %bb.1(0x80000000) + ; HAS-AGPR-NEXT: liveins: $vgpr0 + ; HAS-AGPR-NEXT: {{ $}} + ; HAS-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec + ; HAS-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; HAS-AGPR-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; HAS-AGPR-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec + ; HAS-AGPR-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec + ; HAS-AGPR-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec + ; HAS-AGPR-NEXT: [[AV_MOV_2:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 6, implicit $exec + ; HAS-AGPR-NEXT: {{ $}} + ; HAS-AGPR-NEXT: bb.1: + ; HAS-AGPR-NEXT: [[AV_MOV_3:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 3, implicit $exec + ; + ; NO-AGPR-LABEL: name: func + ; NO-AGPR: bb.0: + ; NO-AGPR-NEXT: successors: %bb.1(0x80000000) + ; NO-AGPR-NEXT: liveins: $vgpr0 + ; NO-AGPR-NEXT: {{ $}} + ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec + ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; NO-AGPR-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; NO-AGPR-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec + ; NO-AGPR-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec + ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec + ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec + ; NO-AGPR-NEXT: {{ $}} + ; NO-AGPR-NEXT: bb.1: + ; NO-AGPR-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec + bb.0: + liveins: $vgpr0 + %0:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec + %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + %3:vgpr_32 = V_MOV_B32_e32 65, implicit $exec + %4:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec + %5:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec + %6:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec + + bb.1: + %7:vgpr_32 = V_MOV_B32_e32 3, implicit $exec + +... + +--- +name: no_agprs +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0 + ; GFX90A-LABEL: name: no_agprs + ; GFX90A: liveins: $vgpr0 + ; GFX90A-NEXT: {{ $}} + ; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; GFX90A-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec + ; + ; GFX908-LABEL: name: no_agprs + ; GFX908: liveins: $vgpr0 + ; GFX908-NEXT: {{ $}} + ; GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec + ; + ; NO-AGPR-LABEL: name: no_agprs + ; NO-AGPR: liveins: $vgpr0 + ; NO-AGPR-NEXT: {{ $}} + ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec + %0:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec + +... diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll new file mode 100644 index 0000000..535f05bc --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll @@ -0,0 +1,122 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX11-TRUE16 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX11-FAKE16 %s + +@global_smem = external local_unnamed_addr addrspace(1) global [0 x i8], align 16 + +define amdgpu_kernel void @v_atomicrmw_fadd_bf16(ptr addrspace(1) %out, i1 %in, ptr addrspace(1) %ptr) #0 { +; GFX11-TRUE16-LABEL: v_atomicrmw_fadd_bf16: +; GFX11-TRUE16: ; %bb.0: +; GFX11-TRUE16-NEXT: s_clause 0x1 +; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 +; GFX11-TRUE16-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0 +; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-TRUE16-NEXT: global_load_b32 v2, v0, s[0:1] offset:4 +; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, -4 +; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3 +; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 3 +; GFX11-TRUE16-NEXT: s_load_b32 s3, s[0:1], 0x0 +; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 3 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h +; GFX11-TRUE16-NEXT: s_lshl_b32 s4, 0xffff, s2 +; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s3 +; GFX11-TRUE16-NEXT: s_not_b32 s3, s4 +; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0 +; GFX11-TRUE16-NEXT: .p2align 6 +; GFX11-TRUE16-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0 +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v0, v4, v[0:1], s[0:1] glc +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: buffer_gl1_inv +; GFX11-TRUE16-NEXT: buffer_gl0_inv +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0 +; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4 +; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1 +; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-TRUE16-NEXT: s_endpgm +; +; GFX11-FAKE16-LABEL: v_atomicrmw_fadd_bf16: +; GFX11-FAKE16: ; %bb.0: +; GFX11-FAKE16-NEXT: s_clause 0x1 +; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 +; GFX11-FAKE16-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 +; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0 +; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[0:1] offset:4 +; GFX11-FAKE16-NEXT: s_and_b32 s0, s2, -4 +; GFX11-FAKE16-NEXT: s_mov_b32 s1, s3 +; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 3 +; GFX11-FAKE16-NEXT: s_load_b32 s3, s[0:1], 0x0 +; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 3 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-FAKE16-NEXT: s_lshl_b32 s4, 0xffff, s2 +; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s3 +; GFX11-FAKE16-NEXT: s_not_b32 s3, s4 +; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0 +; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-FAKE16-NEXT: .p2align 6 +; GFX11-FAKE16-NEXT: .LBB0_1: ; %atomicrmw.start +; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1 +; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1 +; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo +; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v0 +; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc +; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-FAKE16-NEXT: buffer_gl1_inv +; GFX11-FAKE16-NEXT: buffer_gl0_inv +; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 +; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v0 +; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4 +; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4 +; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB0_1 +; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX11-FAKE16-NEXT: s_endpgm + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %in.gep = getelementptr <{ [0 x i8] }>, ptr addrspace(1) %ptr, i64 0, i32 0, i32 %tid + %load = load <4 x bfloat>, ptr addrspace(1) %in.gep + %extract1 = extractelement <4 x bfloat> %load, i64 3 + %fadd = atomicrmw fadd ptr addrspace(1) %out, bfloat %extract1 syncscope("agent") acq_rel + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir new file mode 100644 index 0000000..1a457c9 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir @@ -0,0 +1,74 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s + +--- +name: bad_ra +tracksRegLiveness: true +registers: + - { id: 0, class: sgpr_64, preferred-register: '$sgpr4_sgpr5' } + - { id: 1, class: sgpr_128, preferred-register: '%2' } + - { id: 2, class: areg_128, preferred-register: '%1' } + - { id: 3, class: areg_128, preferred-register: '%4' } + - { id: 4, class: av_128, preferred-register: '%3' } + - { id: 5, class: areg_128, preferred-register: '%6' } + - { id: 6, class: vreg_128, preferred-register: '%5' } + - { id: 7, class: areg_128, preferred-register: '%4' } + - { id: 8, class: vgpr_32 } + - { id: 9, class: vgpr_32 } + - { id: 10, class: vgpr_32 } + - { id: 11, class: areg_128 } +liveins: + - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' } +frameInfo: + maxAlignment: 1 + isCalleeSavedInfoValid: true +machineFunctionInfo: + isEntryFunction: true + scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' + stackPtrOffsetReg: '$sgpr32' + occupancy: 10 + vgprForAGPRCopy: '$vgpr255' + sgprForEXECCopy: '$sgpr100_sgpr101' +body: | + bb.0: + liveins: $sgpr4_sgpr5 + + ; CHECK-LABEL: name: bad_ra + ; CHECK: liveins: $sgpr4_sgpr5 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4) + ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1) + ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 1065353216, implicit $exec + ; CHECK-NEXT: renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec + ; CHECK-NEXT: renamable $vgpr6 = V_MOV_B32_e32 1073741824, implicit $exec + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr1 = COPY renamable $agpr1 + ; CHECK-NEXT: renamable $vgpr0 = COPY renamable $agpr0 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr3 = COPY renamable $agpr1 + ; CHECK-NEXT: renamable $vgpr2 = COPY killed renamable $agpr0 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3 + ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr4, killed $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr5, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1) + ; CHECK-NEXT: S_ENDPGM 0 + early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4) + renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1) + %8:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec + %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + %10:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec + %2:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 + %3:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %2, 0, 0, 0, implicit $mode, implicit $exec + undef %4.sub1:av_128 = COPY %3.sub1 + %4.sub0:av_128 = COPY %3.sub0 + %11:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %3, 0, 0, 0, implicit $mode, implicit $exec + %4.sub3:av_128 = COPY %11.sub1 + %4.sub2:av_128 = COPY %11.sub0 + %7:areg_128 = COPY %4 + %5:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %7, 0, 0, 0, implicit $mode, implicit $exec + %6:vreg_128 = COPY %5 + GLOBAL_STORE_DWORDX4_SADDR %9, %6, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1) + S_ENDPGM 0 + +... diff --git a/llvm/test/CodeGen/AMDGPU/bf16-math.ll b/llvm/test/CodeGen/AMDGPU/bf16-math.ll new file mode 100644 index 0000000..029604c --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/bf16-math.ll @@ -0,0 +1,51 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s + +; TODO: Add global-isel when it can support bf16 + +define amdgpu_ps void @llvm_log2_bf16_v(ptr addrspace(1) %out, bfloat %src) { +; GCN-LABEL: llvm_log2_bf16_v: +; GCN: ; %bb.0: +; GCN-NEXT: v_log_bf16_e32 v2, v2 +; GCN-NEXT: global_store_b16 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %log = call bfloat @llvm.log2.bf16(bfloat %src) + store bfloat %log, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_ps void @llvm_log2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) { +; GCN-LABEL: llvm_log2_bf16_s: +; GCN: ; %bb.0: +; GCN-NEXT: v_log_bf16_e32 v2, s0 +; GCN-NEXT: global_store_b16 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %log = call bfloat @llvm.log2.bf16(bfloat %src) + store bfloat %log, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_ps void @llvm_exp2_bf16_v(ptr addrspace(1) %out, bfloat %src) { +; GCN-LABEL: llvm_exp2_bf16_v: +; GCN: ; %bb.0: +; GCN-NEXT: v_exp_bf16_e32 v2, v2 +; GCN-NEXT: global_store_b16 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %exp = call bfloat @llvm.exp2.bf16(bfloat %src) + store bfloat %exp, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_ps void @llvm_exp2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) { +; GCN-LABEL: llvm_exp2_bf16_s: +; GCN: ; %bb.0: +; GCN-NEXT: v_exp_bf16_e32 v2, s0 +; GCN-NEXT: global_store_b16 v[0:1], v2, off +; GCN-NEXT: s_endpgm + %exp = call bfloat @llvm.exp2.bf16(bfloat %src) + store bfloat %exp, ptr addrspace(1) %out, align 2 + ret void +} + +declare bfloat @llvm.log2.bf16(bfloat) +declare bfloat @llvm.exp2.bf16(bfloat) diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 2bdf994..cd6d741 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -9082,17 +9082,19 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) { ; GFX11TRUE16-LABEL: v_fadd_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -13318,9 +13320,10 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) { ; GFX11TRUE16-LABEL: v_fadd_bf16_fpimm_0: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v0 +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -13413,9 +13416,10 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) { ; GFX11TRUE16-LABEL: v_fadd_bf16_fpimm_1: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 0x42280000, v0 +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 0x42280000, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -13515,17 +13519,19 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) { ; GFX11TRUE16-LABEL: v_fsub_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -14275,17 +14281,19 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) { ; GFX11TRUE16-LABEL: v_fmul_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -18568,32 +18576,34 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) { ; GFX11TRUE16-LABEL: v_fdiv_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0 -; GFX11TRUE16-NEXT: v_rcp_f32_e32 v3, v2 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX11TRUE16-NEXT: v_div_scale_f32 v1, null, v0, v0, v2 +; GFX11TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, v2, v0, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_rcp_f32_e32 v3, v1 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff -; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v3, 1.0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_fma_f32 v4, -v1, v3, 1.0 ; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v4, v3 -; GFX11TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v5, v3 +; GFX11TRUE16-NEXT: v_fma_f32 v6, -v1, v4, v5 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_fma_f32 v6, -v2, v4, v5 ; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v6, v3 +; GFX11TRUE16-NEXT: v_fma_f32 v1, -v1, v4, v5 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_fma_f32 v2, -v2, v4, v5 -; GFX11TRUE16-NEXT: v_div_fmas_f32 v2, v2, v3, v4 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v2, v1, v0 +; GFX11TRUE16-NEXT: v_div_fmas_f32 v1, v1, v3, v4 +; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v1, v0, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -19018,17 +19028,19 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) { ; GFX11TRUE16-LABEL: v_minnum_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -23270,17 +23282,19 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) { ; GFX11TRUE16-LABEL: v_maxnum_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -27591,11 +27605,12 @@ define bfloat @v_sqrt_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_sqrt_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0 -; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v1 +; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x4f800000, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff @@ -27730,9 +27745,10 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) { ; GFX11TRUE16-LABEL: v_ldexp_bf16_i32: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v2, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -27836,17 +27852,18 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) { ; GFX11TRUE16-LABEL: v_frexp_bf16_i16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11TRUE16-NEXT: v_frexp_mant_f32_e32 v0, v1 +; GFX11TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -28019,11 +28036,12 @@ define bfloat @v_log_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_log_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo -; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff @@ -28177,13 +28195,14 @@ define bfloat @v_log2_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_log2_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 ; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo -; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff ; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 @@ -28367,11 +28386,12 @@ define bfloat @v_log10_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_log10_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo -; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff @@ -28580,25 +28600,26 @@ define bfloat @v_exp_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_exp_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0 -; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1 -; GFX11TRUE16-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v1 +; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v1 +; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3fb8aa3b, v1, -v0 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2 -; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3 -; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v1, 0x32a5705f, v2 +; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v3 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3 +; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff -; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo -; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo +; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v0, vcc_lo ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 @@ -28744,13 +28765,14 @@ define bfloat @v_exp2_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_exp2_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 ; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo -; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff ; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 @@ -28937,25 +28959,26 @@ define bfloat @v_exp10_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_exp10_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0 -; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1 -; GFX11TRUE16-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x40549a78, v1 +; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v1 +; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x40549a78, v1, -v0 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2 -; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3 -; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v1, 0x33979a37, v2 +; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v3 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3 +; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0 ; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff -; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo -; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0 -; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo +; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v1 +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v0, vcc_lo ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 @@ -29066,9 +29089,10 @@ define bfloat @v_ceil_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_ceil_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_ceil_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: v_ceil_f32_e32 v0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29163,9 +29187,10 @@ define bfloat @v_trunc_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_trunc_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29260,9 +29285,10 @@ define bfloat @v_rint_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_rint_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29357,9 +29383,10 @@ define bfloat @v_nearbyint_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_nearbyint_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29483,16 +29510,17 @@ define bfloat @v_round_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_round_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_trunc_f32_e32 v1, v0 -; GFX11TRUE16-NEXT: v_sub_f32_e32 v2, v0, v1 +; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v1 +; GFX11TRUE16-NEXT: v_sub_f32_e32 v2, v1, v0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5 ; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0 -; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 @@ -29594,9 +29622,10 @@ define bfloat @v_roundeven_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_roundeven_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29691,9 +29720,10 @@ define bfloat @v_floor_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_floor_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_floor_f32_e32 v0, v0 +; GFX11TRUE16-NEXT: v_floor_f32_e32 v0, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29786,9 +29816,10 @@ define bfloat @v_canonicalize_bf16(bfloat %a) { ; GFX11TRUE16-LABEL: v_canonicalize_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v0 +; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v1, v1 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 @@ -29916,15 +29947,27 @@ define i1 @v_fcmp_oeq_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_oeq_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_oeq_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_oeq_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp oeq bfloat %a, %b ret i1 %op } @@ -29979,15 +30022,27 @@ define i1 @v_fcmp_ogt_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_ogt_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_ogt_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_ogt_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp ogt bfloat %a, %b ret i1 %op } @@ -30042,15 +30097,27 @@ define i1 @v_fcmp_oge_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_oge_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_ge_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_oge_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_ge_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_oge_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_ge_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp oge bfloat %a, %b ret i1 %op } @@ -30105,15 +30172,27 @@ define i1 @v_fcmp_olt_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_olt_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_olt_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_olt_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp olt bfloat %a, %b ret i1 %op } @@ -30168,15 +30247,27 @@ define i1 @v_fcmp_ole_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_ole_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_le_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_ole_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_le_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_ole_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_le_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp ole bfloat %a, %b ret i1 %op } @@ -30231,15 +30322,27 @@ define i1 @v_fcmp_one_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_one_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_lg_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_one_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_one_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp one bfloat %a, %b ret i1 %op } @@ -30294,15 +30397,27 @@ define i1 @v_fcmp_uno_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_uno_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_uno_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_uno_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp uno bfloat %a, %b ret i1 %op } @@ -30357,15 +30472,27 @@ define i1 @v_fcmp_ueq_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_ueq_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_ueq_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_ueq_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp ueq bfloat %a, %b ret i1 %op } @@ -30420,15 +30547,27 @@ define i1 @v_fcmp_ugt_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_ugt_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_nle_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_ugt_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_nle_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_ugt_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_nle_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp ugt bfloat %a, %b ret i1 %op } @@ -30483,15 +30622,27 @@ define i1 @v_fcmp_uge_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_uge_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_uge_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_uge_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp uge bfloat %a, %b ret i1 %op } @@ -30546,15 +30697,27 @@ define i1 @v_fcmp_ult_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_ult_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_nge_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_ult_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_nge_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_ult_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_nge_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp ult bfloat %a, %b ret i1 %op } @@ -30609,15 +30772,27 @@ define i1 @v_fcmp_ule_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_ule_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_ule_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_ule_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp ule bfloat %a, %b ret i1 %op } @@ -30672,15 +30847,27 @@ define i1 @v_fcmp_une_bf16(bfloat %a, bfloat %b) { ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fcmp_une_bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_neq_f32_e32 vcc_lo, v0, v1 -; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fcmp_une_bf16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11TRUE16-NEXT: v_cmp_neq_f32_e32 vcc_lo, v1, v2 +; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fcmp_une_bf16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cmp_neq_f32_e32 vcc_lo, v0, v1 +; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fcmp une bfloat %a, %b ret i1 %op } @@ -30763,13 +30950,22 @@ define i16 @v_fptosi_bf16_to_i16(bfloat %x) { ; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fptosi_bf16_to_i16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cvt_i32_f32_e32 v0, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fptosi_bf16_to_i16: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v1 +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fptosi_bf16_to_i16: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v0, v0 +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fptosi bfloat %x to i16 ret i16 %op } @@ -31144,13 +31340,22 @@ define i32 @v_fptosi_bf16_to_i32(bfloat %x) { ; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fptosi_bf16_to_i32: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cvt_i32_f32_e32 v0, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fptosi_bf16_to_i32: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v1 +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fptosi_bf16_to_i32: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v0, v0 +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fptosi bfloat %x to i32 ret i32 %op } @@ -31494,27 +31699,50 @@ define i64 @v_fptosi_bf16_to_i64(bfloat %x) { ; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: v_fptosi_bf16_to_i64: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_trunc_f32_e32 v0, v0 -; GFX11-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0| -; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_floor_f32_e32 v1, v1 -; GFX11-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0| -; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v2 -; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_xor_b32_e32 v0, v0, v3 -; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11TRUE16-LABEL: v_fptosi_bf16_to_i64: +; GFX11TRUE16: ; %bb.0: +; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v1 +; GFX11TRUE16-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0| +; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_floor_f32_e32 v1, v1 +; GFX11TRUE16-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0| +; GFX11TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v2 +; GFX11TRUE16-NEXT: v_xor_b32_e32 v1, v1, v3 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_xor_b32_e32 v0, v0, v3 +; GFX11TRUE16-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo +; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11FAKE16-LABEL: v_fptosi_bf16_to_i64: +; GFX11FAKE16: ; %bb.0: +; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_trunc_f32_e32 v0, v0 +; GFX11FAKE16-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0| +; GFX11FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v0 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_floor_f32_e32 v1, v1 +; GFX11FAKE16-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0| +; GFX11FAKE16-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11FAKE16-NEXT: v_cvt_u32_f32_e32 v0, v2 +; GFX11FAKE16-NEXT: v_xor_b32_e32 v1, v1, v3 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_xor_b32_e32 v0, v0, v3 +; GFX11FAKE16-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3 +; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11FAKE16-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo +; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31] %op = fptosi bfloat %x to i64 ret i64 %op } @@ -42575,18 +42803,21 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) { ; GFX11TRUE16-LABEL: v_fma_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v1, v2 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1 +; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3 +; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v0, v1 -; GFX11TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1 -; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2 -; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -43457,26 +43688,30 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) { ; GFX11TRUE16-LABEL: v_fmuladd_bf16: ; GFX11TRUE16: ; %bb.0: ; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v1, v3 +; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 -; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v0, v1, v3 :: v_dual_lshlrev_b32 v1, 16, v2 ; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo ; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1 -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 ; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff -; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll index ae90cfb..7eb7d72 100644 --- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll +++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll @@ -25,7 +25,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr17, 8, implicit-def $scc ; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit killed $scc ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc - ; GFX90A-NEXT: renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $vgpr2 = DS_READ_B32_gfx9 renamable $vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3) ; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_MOV_B64 0 ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc @@ -56,8 +56,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc - ; GFX90A-NEXT: renamable $vgpr15 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr17 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr15 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr17 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: S_CBRANCH_VCCZ %bb.57, implicit $vcc ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: bb.4.bb15: @@ -112,14 +112,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: successors: %bb.7(0x80000000) ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3 ; GFX90A-NEXT: {{ $}} - ; GFX90A-NEXT: renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr21 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr20 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr23 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr22 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr25 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr24 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr19 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr18 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr21 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr25 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: bb.7.Flow19: ; GFX90A-NEXT: successors: %bb.62(0x40000000), %bb.8(0x40000000) @@ -671,7 +671,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57 ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec - ; GFX90A-NEXT: renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr50_sgpr51, implicit-def dead $scc ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.54, implicit $vcc ; GFX90A-NEXT: {{ $}} @@ -759,7 +759,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3 ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: renamable $vgpr53 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec - ; GFX90A-NEXT: renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr10 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $vgpr14_vgpr15 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3) ; GFX90A-NEXT: renamable $vgpr10 = COPY renamable $sgpr21, implicit $exec ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3) @@ -801,12 +801,12 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: renamable $vgpr42_vgpr43 = IMPLICIT_DEF ; GFX90A-NEXT: renamable $vgpr40_vgpr41 = IMPLICIT_DEF ; GFX90A-NEXT: renamable $vgpr46_vgpr47 = IMPLICIT_DEF - ; GFX90A-NEXT: renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr52 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr16 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr53 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr52 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr16 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr53 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr13 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr12 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 0 ; GFX90A-NEXT: S_BRANCH %bb.7 ; GFX90A-NEXT: {{ $}} @@ -814,7 +814,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: successors: %bb.3(0x80000000) ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3 ; GFX90A-NEXT: {{ $}} - ; GFX90A-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3) ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec ; GFX90A-NEXT: renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3) @@ -913,7 +913,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr25, implicit $exec ; GFX90A-NEXT: renamable $vgpr3 = V_OR_B32_e32 killed $vgpr11, killed $vgpr19, implicit $exec ; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec - ; GFX90A-NEXT: renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U32_sdwa 0, killed $vgpr53, 0, $vgpr3, 0, 0, 6, implicit $exec ; GFX90A-NEXT: renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec ; GFX90A-NEXT: renamable $vgpr10 = V_OR_B32_e32 killed $vgpr52, killed $vgpr13, implicit $exec @@ -955,7 +955,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: renamable $vgpr10 = COPY renamable $sgpr27, implicit $exec ; GFX90A-NEXT: renamable $vgpr2, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr2, 0, implicit $exec ; GFX90A-NEXT: renamable $vgpr3, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr10, killed $vgpr3, killed $vcc, 0, implicit $exec - ; GFX90A-NEXT: renamable $vgpr27 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr27 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: renamable $vgpr49 = COPY renamable $vgpr27, implicit $exec ; GFX90A-NEXT: renamable $vgpr35 = COPY renamable $vgpr27, implicit $exec ; GFX90A-NEXT: renamable $vgpr39 = COPY renamable $vgpr27, implicit $exec @@ -989,7 +989,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: renamable $vgpr10 = V_OR_B32_e32 $vgpr50, killed $vgpr16, implicit $exec ; GFX90A-NEXT: renamable $vgpr54 = V_OR_B32_e32 killed $vgpr10, killed $vgpr14, implicit $exec - ; GFX90A-NEXT: renamable $vgpr55 = V_MOV_B32_e32 0, implicit $exec + ; GFX90A-NEXT: renamable $vgpr55 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr55, renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3) ; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_MOV_B64 0 ; GFX90A-NEXT: S_BRANCH %bb.69 diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll index 348862d..f4b432d 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll @@ -5100,55 +5100,56 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4 ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen ; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1 +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3 ; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -5257,53 +5258,54 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4 ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen ; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3 +; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1 +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory: @@ -5619,48 +5621,49 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4 ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen ; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, v1, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4 ; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 @@ -5773,46 +5776,47 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4 ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen ; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4 +; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 @@ -6124,15 +6128,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4 +; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7 ; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1 @@ -6146,39 +6150,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen ; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0 ; GFX12-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l ; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff +; GFX12-TRUE16-NEXT: v_add_f32_e32 v6, v6, v8 +; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5 -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6 +; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6 +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX12-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1 ; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 @@ -6193,14 +6196,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_4 ; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1 ; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -6208,7 +6211,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3 ; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -6384,16 +6387,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0 ; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7 ; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1 @@ -6405,39 +6408,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 -; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen ; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l ; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h +; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5 -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6 +; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6 +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX11-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1 ; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 @@ -6451,14 +6453,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc ; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_4 ; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1 ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8 ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv ; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1 @@ -6468,7 +6470,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory: diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll index ab867b0..6f1675e 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll @@ -4228,55 +4228,56 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4 ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen ; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v1, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1 +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3 ; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4385,53 +4386,54 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4 ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen ; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3 +; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1 +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory: @@ -4749,48 +4751,49 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4 ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen ; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v1, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4 ; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 @@ -4903,46 +4906,47 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4 ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen ; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v1, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4 +; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 @@ -5256,15 +5260,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4 +; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7 ; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1 @@ -5278,39 +5282,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen ; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0 ; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l ; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v10 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v8 +; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5 -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6 +; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6 +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1 ; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 @@ -5325,14 +5328,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4 ; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1 ; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -5340,7 +5343,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3 ; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -5516,16 +5519,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0 ; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7 ; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1 @@ -5537,39 +5540,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 -; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen ; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l ; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h +; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5 -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6 +; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6 +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1 ; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 @@ -5583,14 +5585,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc ; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4 ; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1 ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8 ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv ; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1 @@ -5600,7 +5602,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory: diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll index 1a25904..acb27be 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll @@ -4228,55 +4228,56 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4 ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen ; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v1, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1 +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3 ; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4385,53 +4386,54 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4 ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen ; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v1, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3 +; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1 +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory: @@ -4749,48 +4751,49 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4 ; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen ; GFX12-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v1, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe -; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4 ; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 @@ -4903,46 +4906,47 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4 ; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4 -; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen ; GFX11-TRUE16-NEXT: s_not_b32 s6, s5 ; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v1, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4 -; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4 +; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0 -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4 ; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5 @@ -5256,15 +5260,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4 +; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff -; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7 ; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1 @@ -5278,39 +5282,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen +; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen ; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1 ; GFX12-TRUE16-NEXT: ; %bb.2: ; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0 ; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l ; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v10 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v8 +; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5 -; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6 +; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6 +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1 ; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2 ; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 @@ -5325,14 +5328,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN +; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN ; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4 ; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1 ; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -5340,7 +5343,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3 ; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end ; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4 +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -5516,16 +5519,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4 +; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0 ; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7 ; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1 @@ -5537,39 +5540,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 -; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen +; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen ; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1 ; GFX11-TRUE16-NEXT: ; %bb.2: ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l ; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h +; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v8 +; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5 -; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6 +; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6 +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6 ; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1 ; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2 ; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0 @@ -5583,14 +5585,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc +; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc ; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0 ; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4 ; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1 ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8 ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv ; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1 @@ -5600,7 +5602,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd ; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end ; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2 ; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1 -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory: diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll index c69e127..3c991cf 100644 --- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll +++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll @@ -444,14 +444,6 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7) ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v62, s[8:11], 0 offen ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v62, s[8:11], 0 offen offset:16 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v62, s[8:11], 0 offen offset:32 -; GISEL-GFX942-NEXT: v_add_u32_e32 v63, s12, v1 -; GISEL-GFX942-NEXT: v_add_u32_e32 v1, 0x100, v1 -; GISEL-GFX942-NEXT: v_cmp_lt_u32_e32 vcc, v1, v0 -; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0) -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a0, v13 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v12 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v11 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a3, v10 ; Reload Reuse ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[14:17], v62, s[8:11], 0 offen offset:48 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[18:21], v62, s[8:11], 0 offen offset:64 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[22:25], v62, s[8:11], 0 offen offset:80 @@ -464,20 +456,15 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7) ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[50:53], v62, s[8:11], 0 offen offset:192 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v62, s[8:11], 0 offen offset:208 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v62, s[8:11], 0 offen offset:224 -; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v62, s[8:11], 0 offen offset:240 -; GISEL-GFX942-NEXT: s_nop 0 +; GISEL-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v62, s[8:11], 0 offen offset:240 +; GISEL-GFX942-NEXT: v_add_u32_e32 v63, s12, v1 +; GISEL-GFX942-NEXT: v_add_u32_e32 v1, 0x100, v1 +; GISEL-GFX942-NEXT: v_cmp_lt_u32_e32 vcc, v1, v0 +; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0) +; GISEL-GFX942-NEXT: scratch_store_dwordx4 off, a[0:3], off ; 16-byte Folded Spill ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v63, s[4:7], 0 offen ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[6:9], v63, s[4:7], 0 offen offset:16 -; GISEL-GFX942-NEXT: s_waitcnt vmcnt(2) -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a4, v13 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a5, v12 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a6, v11 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a7, v10 ; Reload Reuse -; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v63, s[4:7], 0 offen offset:32 +; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[10:13], v63, s[4:7], 0 offen offset:32 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[14:17], v63, s[4:7], 0 offen offset:48 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[18:21], v63, s[4:7], 0 offen offset:64 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[22:25], v63, s[4:7], 0 offen offset:80 @@ -490,10 +477,8 @@ define amdgpu_kernel void @memcpy_known(ptr addrspace(7) %src, ptr addrspace(7) ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[50:53], v63, s[4:7], 0 offen offset:192 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[54:57], v63, s[4:7], 0 offen offset:208 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[58:61], v63, s[4:7], 0 offen offset:224 -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a4 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a5 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a6 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a7 ; Reload Reuse +; GISEL-GFX942-NEXT: scratch_load_dwordx4 v[2:5], off, off ; 16-byte Folded Reload +; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0) ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v63, s[4:7], 0 offen offset:240 ; GISEL-GFX942-NEXT: s_cbranch_vccnz .LBB0_1 ; GISEL-GFX942-NEXT: ; %bb.2: ; %memcpy-split @@ -822,14 +807,6 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v1, s[4:7], 0 offen ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v1, s[4:7], 0 offen offset:16 ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v1, s[4:7], 0 offen offset:32 -; SDAG-GFX942-NEXT: v_add_u32_e32 v62, s8, v0 -; SDAG-GFX942-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v0 -; SDAG-GFX942-NEXT: s_and_b64 vcc, exec, vcc -; SDAG-GFX942-NEXT: s_waitcnt vmcnt(0) -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a0, v13 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a1, v12 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a2, v11 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a3, v10 ; Reload Reuse ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[14:17], v1, s[4:7], 0 offen offset:48 ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[18:21], v1, s[4:7], 0 offen offset:64 ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[22:25], v1, s[4:7], 0 offen offset:80 @@ -842,20 +819,16 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[50:53], v1, s[4:7], 0 offen offset:192 ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v1, s[4:7], 0 offen offset:208 ; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v1, s[4:7], 0 offen offset:224 -; SDAG-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v1, s[4:7], 0 offen offset:240 -; SDAG-GFX942-NEXT: s_nop 0 +; SDAG-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v1, s[4:7], 0 offen offset:240 +; SDAG-GFX942-NEXT: v_add_u32_e32 v62, s8, v0 +; SDAG-GFX942-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v0 +; SDAG-GFX942-NEXT: s_and_b64 vcc, exec, vcc +; SDAG-GFX942-NEXT: s_waitcnt vmcnt(0) +; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v63, a3 ; Reload Reuse +; SDAG-GFX942-NEXT: scratch_store_dwordx3 off, a[0:2], off ; 12-byte Folded Spill ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v62, s[12:15], 0 offen ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[6:9], v62, s[12:15], 0 offen offset:16 -; SDAG-GFX942-NEXT: s_waitcnt vmcnt(2) -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a4, v13 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a5, v12 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a6, v11 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_write_b32 a7, v10 ; Reload Reuse -; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v62, s[12:15], 0 offen offset:32 +; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[10:13], v62, s[12:15], 0 offen offset:32 ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[14:17], v62, s[12:15], 0 offen offset:48 ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[18:21], v62, s[12:15], 0 offen offset:64 ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[22:25], v62, s[12:15], 0 offen offset:80 @@ -868,10 +841,8 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[50:53], v62, s[12:15], 0 offen offset:192 ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[54:57], v62, s[12:15], 0 offen offset:208 ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[58:61], v62, s[12:15], 0 offen offset:224 -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v5, a4 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v4, a5 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v3, a6 ; Reload Reuse -; SDAG-GFX942-NEXT: v_accvgpr_read_b32 v2, a7 ; Reload Reuse +; SDAG-GFX942-NEXT: scratch_load_dwordx3 v[2:4], off, off ; 12-byte Folded Reload +; SDAG-GFX942-NEXT: s_waitcnt vmcnt(0) ; SDAG-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v62, s[12:15], 0 offen offset:240 ; SDAG-GFX942-NEXT: s_cbranch_vccnz .LBB1_1 ; SDAG-GFX942-NEXT: ; %bb.2: ; %memcpy-split @@ -993,16 +964,6 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[2:5], v1, s[8:11], 0 offen ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[6:9], v1, s[8:11], 0 offen offset:16 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v1, s[8:11], 0 offen offset:32 -; GISEL-GFX942-NEXT: v_add_u32_e32 v62, s12, v0 -; GISEL-GFX942-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v0 -; GISEL-GFX942-NEXT: s_xor_b64 s[2:3], vcc, -1 -; GISEL-GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], -1 -; GISEL-GFX942-NEXT: s_and_b64 vcc, s[2:3], exec -; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0) -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a0, v13 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a1, v12 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a2, v11 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a3, v10 ; Reload Reuse ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[14:17], v1, s[8:11], 0 offen offset:48 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[18:21], v1, s[8:11], 0 offen offset:64 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[22:25], v1, s[8:11], 0 offen offset:80 @@ -1015,20 +976,18 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[50:53], v1, s[8:11], 0 offen offset:192 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[54:57], v1, s[8:11], 0 offen offset:208 ; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[58:61], v1, s[8:11], 0 offen offset:224 -; GISEL-GFX942-NEXT: buffer_load_dwordx4 v[10:13], v1, s[8:11], 0 offen offset:240 -; GISEL-GFX942-NEXT: s_nop 0 +; GISEL-GFX942-NEXT: buffer_load_dwordx4 a[0:3], v1, s[8:11], 0 offen offset:240 +; GISEL-GFX942-NEXT: v_add_u32_e32 v62, s12, v0 +; GISEL-GFX942-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v0 +; GISEL-GFX942-NEXT: s_xor_b64 s[2:3], vcc, -1 +; GISEL-GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], -1 +; GISEL-GFX942-NEXT: s_and_b64 vcc, s[2:3], exec +; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0) +; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v63, a3 ; Reload Reuse +; GISEL-GFX942-NEXT: scratch_store_dwordx3 off, a[0:2], off ; 12-byte Folded Spill ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v62, s[4:7], 0 offen ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[6:9], v62, s[4:7], 0 offen offset:16 -; GISEL-GFX942-NEXT: s_waitcnt vmcnt(2) -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a4, v13 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a0 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a1 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a2 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a3 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a5, v12 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a6, v11 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_write_b32 a7, v10 ; Reload Reuse -; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v62, s[4:7], 0 offen offset:32 +; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[10:13], v62, s[4:7], 0 offen offset:32 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[14:17], v62, s[4:7], 0 offen offset:48 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[18:21], v62, s[4:7], 0 offen offset:64 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[22:25], v62, s[4:7], 0 offen offset:80 @@ -1041,10 +1000,8 @@ define amdgpu_kernel void @memcpy_known_medium(ptr addrspace(7) %src, ptr addrsp ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[50:53], v62, s[4:7], 0 offen offset:192 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[54:57], v62, s[4:7], 0 offen offset:208 ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[58:61], v62, s[4:7], 0 offen offset:224 -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v5, a4 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v4, a5 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v3, a6 ; Reload Reuse -; GISEL-GFX942-NEXT: v_accvgpr_read_b32 v2, a7 ; Reload Reuse +; GISEL-GFX942-NEXT: scratch_load_dwordx3 v[2:4], off, off ; 12-byte Folded Reload +; GISEL-GFX942-NEXT: s_waitcnt vmcnt(0) ; GISEL-GFX942-NEXT: buffer_store_dwordx4 v[2:5], v62, s[4:7], 0 offen offset:240 ; GISEL-GFX942-NEXT: s_cbranch_vccnz .LBB1_1 ; GISEL-GFX942-NEXT: ; %bb.2: ; %memcpy-split diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll index a1aef8d..da49140 100644 --- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll +++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll @@ -2748,100 +2748,101 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) { ; GFX11-TRUE16-NEXT: v_and_b16 v10.l, v10.l, 1 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 1, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, v8.l, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v6.l, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 2, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v7.l -; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v9.h, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1 ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v6.l, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v5.h, v9.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1 -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v28.l, 1 -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v26.l, 1 -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v3.l, 15 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 1, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 2, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v25.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, v24.l, 1 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 1, v21.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v3.h, v4.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v5.l, v4.h -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, v22.l, 1 -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v20.l, 1 -; GFX11-TRUE16-NEXT: v_and_b16 v6.h, v18.l, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 1, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v16.l, 1 ; GFX11-TRUE16-NEXT: v_and_b16 v14.l, v14.l, 1 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 1, v13.l ; GFX11-TRUE16-NEXT: v_and_b16 v12.l, v12.l, 1 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v30.l, 1 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v23.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 2, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v5.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v19.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 2, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 3, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 2, v14.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v11.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 2, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v8.l, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 3, v7.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.l, v5.h, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v7.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v12.h, v13.h -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, v12.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v25.h, v26.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 1, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, v24.l, 1 +; GFX11-TRUE16-NEXT: v_and_b16 v22.l, v22.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 1, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v20.l, v20.l, 1 +; GFX11-TRUE16-NEXT: v_and_b16 v18.l, v18.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 1, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v16.l, v16.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 3, v15.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 2, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v12.l, v13.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v10.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v5.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v7.l, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3 ; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v3.l, 3 -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v3.h -; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v5.l, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v5.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.h, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3 +; GFX11-TRUE16-NEXT: v_and_b16 v30.l, v30.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 1, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v28.l, v28.l, 1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 3, v27.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 2, v25.h +; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v24.l, v25.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.l, 3, v23.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 2, v22.l +; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v20.l, v21.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 3, v19.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.l, 2, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.h, v10.h +; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v8.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 3, v31.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 2, v30.l +; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v28.l, v29.l +; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v22.h, v25.h +; GFX11-TRUE16-NEXT: v_and_b16 v24.l, v24.l, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v23.l, v22.l +; GFX11-TRUE16-NEXT: v_and_b16 v14.h, v16.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.h, v18.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v15.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 15 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15 -; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v3.l, 15 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 4, v3.h -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 15 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 12, v8.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v30.h, v28.h +; GFX11-TRUE16-NEXT: v_and_b16 v24.h, v24.h, 3 +; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v24.l, v22.h +; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v14.h, v18.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v1.h, v16.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 12, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 12, v1.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l +; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v24.h, v28.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v20.h, 15 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 4, v2.h +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 15 +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 12, v23.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v1.h, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: global_store_b32 v[0:1], v0, off ; GFX11-TRUE16-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll index 745e047..86e890b 100644 --- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll +++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll @@ -1771,33 +1771,35 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 ; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1] ; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v4.l, 9 -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 9 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff00, v4.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff00, v4.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff00, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff00, v4.h ; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte3_e32 v3, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h -; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte2_e32 v2, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l +; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v4.h, 9 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, 0x900, v0.l -; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, 0x900, v0.h +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.h ; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte1_e32 v1, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x900, v0.l +; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte2_e32 v2, v4 ; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v0, v4 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v6, v7 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v7, v6 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: global_store_b128 v5, v[0:3], s[0:1] diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll index 738bad7..f26b720 100644 --- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll +++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll @@ -2811,20 +2811,20 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -2852,20 +2852,20 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -2945,20 +2945,20 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3f00, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -2986,20 +2986,20 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3f00, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3105,34 +3105,34 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a ; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test3: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x4000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x4000, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x4000, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x4000, s0 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: fmul_select_v2bf16_test3: @@ -3170,34 +3170,34 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a ; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test3: ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80 -; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3 -; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x4000, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x4000, s0 +; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80 +; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 +; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x4000, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x4000, s0 ; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1 -; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 -; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff -; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3 +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-GISEL-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-FAKE16-LABEL: fmul_select_v2bf16_test3: @@ -3314,34 +3314,34 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a ; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test4: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x3f00, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x3f00, s0 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1 -; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: fmul_select_v2bf16_test4: @@ -3379,34 +3379,34 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a ; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test4: ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80 -; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3 -; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0 +; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80 +; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 +; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x3f00, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x3f00, s0 ; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1 -; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 -; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 -; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff -; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3 +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h -; GFX11-GISEL-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-FAKE16-LABEL: fmul_select_v2bf16_test4: @@ -3498,20 +3498,20 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3539,20 +3539,20 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3634,20 +3634,20 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc100, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3675,20 +3675,20 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc100, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3769,20 +3769,20 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4100, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3810,20 +3810,20 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4100, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -3902,12 +3902,13 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test8: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 0x8000, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v3, v0 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 @@ -3940,12 +3941,13 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test8: ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 0x8000, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v3, v0 ; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 @@ -4033,20 +4035,20 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc180, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4074,20 +4076,20 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2) ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc180, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4169,20 +4171,20 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xe000, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4210,20 +4212,20 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xe000, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4305,20 +4307,20 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3480, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -4346,20 +4348,20 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b ; GFX11-GISEL-TRUE16: ; %bb.0: ; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3480, vcc_lo +; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 ; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo -; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll index f8e13fc..4cb0d2d 100644 --- a/llvm/test/CodeGen/AMDGPU/div_i128.ll +++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll @@ -521,16 +521,19 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 ; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[12:13] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[12:13] ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[12:13] ; GFX9-O0-NEXT: v_mov_b32_e32 v3, s10 ; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[8:9] ; GFX9-O0-NEXT: ; implicit-def: $sgpr8 @@ -2710,16 +2713,19 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 ; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[12:13] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[12:13] ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[12:13] ; GFX9-O0-NEXT: v_mov_b32_e32 v3, s10 ; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[8:9] ; GFX9-O0-NEXT: ; implicit-def: $sgpr8 diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll index 45fe2d0..85e56a2 100644 --- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll +++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll @@ -77,11 +77,20 @@ define i32 @divergent_vec_0_i16(i16 %a) { ; GFX906-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX906-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: divergent_vec_0_i16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-TRUE16-LABEL: divergent_vec_0_i16: +; GFX11-TRUE16: ; %bb.0: +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1 +; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-FAKE16-LABEL: divergent_vec_0_i16: +; GFX11-FAKE16: ; %bb.0: +; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %tmp = insertelement <2 x i16> poison, i16 0, i32 0 %vec = insertelement <2 x i16> %tmp, i16 %a, i32 1 %val = bitcast <2 x i16> %vec to i32 @@ -160,11 +169,20 @@ define i32 @divergent_vec_i16_0(i16 %a) { ; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX906-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: divergent_vec_i16_0: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-TRUE16-LABEL: divergent_vec_i16_0: +; GFX11-TRUE16: ; %bb.0: +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1 +; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-FAKE16-LABEL: divergent_vec_i16_0: +; GFX11-FAKE16: ; %bb.0: +; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %tmp = insertelement <2 x i16> poison, i16 %a, i32 0 %vec = insertelement <2 x i16> %tmp, i16 0, i32 1 %val = bitcast <2 x i16> %vec to i32 @@ -243,11 +261,20 @@ define float @divergent_vec_f16_0(half %a) { ; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX906-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-LABEL: divergent_vec_f16_0: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-NEXT: s_setpc_b64 s[30:31] +; GFX11-TRUE16-LABEL: divergent_vec_f16_0: +; GFX11-TRUE16: ; %bb.0: +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1 +; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-FAKE16-LABEL: divergent_vec_f16_0: +; GFX11-FAKE16: ; %bb.0: +; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %tmp = insertelement <2 x half> poison, half %a, i32 0 %vec = insertelement <2 x half> %tmp, half 0.0, i32 1 %val = bitcast <2 x half> %vec to float diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll index 6cd4399..d8f81db 100644 --- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll @@ -624,31 +624,30 @@ define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr ad ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3] +; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3] ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v2.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v3 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v1, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_dual_mul_f32 v2, v1, v2 :: v_dual_lshlrev_b32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v3 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0 ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo @@ -809,35 +808,35 @@ define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa ; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 ; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3] -; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s4, 16 -; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, s2, v0 :: v_dual_lshlrev_b32 v1, 16, v1 ; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, 0xffff0000 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) -; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, s2, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, s2, v1 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l +; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s4, 16 +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, s2, v1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h ; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 ; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-TRUE16-NEXT: s_endpgm @@ -988,34 +987,36 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[0:1] ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, 4.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 2.0, v1 -; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, 4.0, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 2.0, v1 +; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff -; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo -; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc -; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo ; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc +; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: s_endpgm ; ; GFX11-FAKE16-LABEL: v_extract_fabs_fold_v2bf16: diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll index 8581e4d03..8c7d5cf 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll @@ -11974,7 +11974,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -11988,20 +11988,22 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12121,7 +12123,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -12136,19 +12138,21 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12425,34 +12429,34 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12578,12 +12582,11 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -12594,19 +12597,21 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12891,34 +12896,34 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13045,12 +13050,11 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -13061,19 +13065,21 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13355,45 +13361,45 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB49_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13505,45 +13511,46 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB49_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -13806,45 +13813,45 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13957,45 +13964,46 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -14259,27 +14267,28 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -14379,27 +14388,28 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) @@ -14629,32 +14639,33 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -14744,33 +14755,34 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -14988,7 +15000,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -15002,18 +15014,20 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -15130,7 +15144,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -15145,17 +15159,19 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -15424,34 +15440,34 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -15579,12 +15595,11 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -15595,19 +15610,21 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -15891,46 +15908,46 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -16043,45 +16060,46 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll index 883063b..56ad91d 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll @@ -9836,7 +9836,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9850,20 +9850,22 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9983,7 +9985,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9998,19 +10000,21 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10288,34 +10292,34 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10441,12 +10445,11 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10457,19 +10460,21 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10755,34 +10760,34 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10909,12 +10914,11 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10925,19 +10929,21 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -11220,7 +11226,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -11234,18 +11240,20 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -11362,7 +11370,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -11377,17 +11385,19 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -11654,45 +11664,45 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11804,45 +11814,46 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12106,45 +12117,45 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -12257,45 +12268,46 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12560,27 +12572,28 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -12680,27 +12693,28 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) @@ -12931,32 +12945,33 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13046,33 +13061,34 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -13294,34 +13310,34 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13449,12 +13465,11 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -13465,19 +13480,21 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13762,46 +13779,46 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13914,45 +13931,46 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll index c603421..f0083bd 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll @@ -9836,7 +9836,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9850,20 +9850,22 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9983,7 +9985,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9998,19 +10000,21 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10288,34 +10292,34 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10441,12 +10445,11 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10457,19 +10460,21 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10755,34 +10760,34 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10909,12 +10914,11 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10925,19 +10929,21 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -11220,7 +11226,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -11234,18 +11240,20 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -11362,7 +11370,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -11377,17 +11385,19 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -11654,45 +11664,45 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11804,45 +11814,46 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12106,45 +12117,45 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -12257,45 +12268,46 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12560,27 +12572,28 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -12680,27 +12693,28 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) @@ -12931,32 +12945,33 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13046,33 +13061,34 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -13294,34 +13310,34 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13449,12 +13465,11 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -13465,19 +13480,21 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13762,46 +13779,46 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13914,45 +13931,46 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g ; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll index c987effe..3ee0bb2 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll @@ -9419,7 +9419,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9433,20 +9433,22 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9566,7 +9568,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9581,19 +9583,21 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9870,34 +9874,34 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat % ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10023,12 +10027,11 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat % ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10039,19 +10042,21 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat % ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10336,34 +10341,34 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat % ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10490,12 +10495,11 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat % ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10506,19 +10510,21 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat % ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10800,7 +10806,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10814,18 +10820,20 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -10942,7 +10950,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10957,17 +10965,19 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 { ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -11233,45 +11243,45 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat % ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11383,45 +11393,46 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat % ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b_pos: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -11684,45 +11695,45 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat % ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11835,45 +11846,46 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat % ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b_neg: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12137,27 +12149,28 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr, ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 @@ -12257,27 +12270,28 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr, ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) @@ -12507,32 +12521,33 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr, ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -12622,33 +12637,34 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr, ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12869,34 +12885,34 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13024,12 +13040,11 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -13040,19 +13055,21 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13336,46 +13353,46 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13488,45 +13505,46 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat ; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_noret_bf16__offset12b_pos: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc +; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll new file mode 100644 index 0000000..f4040f3 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll @@ -0,0 +1,6030 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s + +; Test using saddr addressing mode of flat_* atomic instructions. + +define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xchg_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +; Maximum positive offset on gfx10 +define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_2047: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:2047 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 2047 + %unused = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +; Maximum negative offset on gfx10 +define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:-2048 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -2048 + %unused = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xchg_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_2048: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 2048 + %rtn = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_neg2048: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -2048 + %rtn = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +; -------------------------------------------------------------------------------- +; Uniformity edge cases +; -------------------------------------------------------------------------------- + +@ptr.in.lds = internal addrspace(3) global ptr undef + +; Base pointer is uniform, but also in VGPRs +define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i32 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +; Base pointer is uniform, but also in VGPRs, with imm offset +define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %voffset, i32 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] offset:42 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 offset:42 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42 + %rtn = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +; Base pointer is uniform, but also in VGPRs +define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset, i32 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_endpgm + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +; Base pointer is uniform, but also in VGPRs, with imm offset +define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32 %voffset, i32 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v1, s[0:1] offset:42 scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 offset:42 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_endpgm + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42 + %unused = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; All atomicrmw ops +; -------------------------------------------------------------------------------- + +; -------------------------------------------------------------------------------- +; atomicrmw xchg +; -------------------------------------------------------------------------------- + +define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB10_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB10_4 +; GFX1250-SDAG-NEXT: .LBB10_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB10_5 +; GFX1250-SDAG-NEXT: .LBB10_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB10_2 +; GFX1250-SDAG-NEXT: .LBB10_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB10_5 +; GFX1250-SDAG-NEXT: .LBB10_5: +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB10_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB10_4 +; GFX1250-GISEL-NEXT: .LBB10_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB10_5 +; GFX1250-GISEL-NEXT: .LBB10_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB10_2 +; GFX1250-GISEL-NEXT: .LBB10_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[4:5], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB10_5 +; GFX1250-GISEL-NEXT: .LBB10_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw xchg ptr %gep0, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB11_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB11_4 +; GFX1250-SDAG-NEXT: .LBB11_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB11_5 +; GFX1250-SDAG-NEXT: .LBB11_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB11_2 +; GFX1250-SDAG-NEXT: .LBB11_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB11_5 +; GFX1250-SDAG-NEXT: .LBB11_5: +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB11_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB11_4 +; GFX1250-GISEL-NEXT: .LBB11_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB11_5 +; GFX1250-GISEL-NEXT: .LBB11_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB11_2 +; GFX1250-GISEL-NEXT: .LBB11_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[4:5], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB11_5 +; GFX1250-GISEL-NEXT: .LBB11_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw xchg ptr %gep1, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB12_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB12_4 +; GFX1250-SDAG-NEXT: .LBB12_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB12_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB12_2 +; GFX1250-SDAG-NEXT: .LBB12_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_store_b64 v0, v[2:3], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB12_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB12_4 +; GFX1250-GISEL-NEXT: .LBB12_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB12_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB12_2 +; GFX1250-GISEL-NEXT: .LBB12_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v0, v[4:5], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw xchg ptr %gep0, i64 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB13_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB13_4 +; GFX1250-SDAG-NEXT: .LBB13_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB13_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB13_2 +; GFX1250-SDAG-NEXT: .LBB13_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_store_b64 v0, v[2:3], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB13_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB13_4 +; GFX1250-GISEL-NEXT: .LBB13_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB13_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB13_2 +; GFX1250-GISEL-NEXT: .LBB13_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v0, v[4:5], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw xchg ptr %gep1, i64 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw add +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_add_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw add ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_add_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw add ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_add_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw add ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_add_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw add ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB18_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB18_4 +; GFX1250-SDAG-NEXT: .LBB18_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB18_5 +; GFX1250-SDAG-NEXT: .LBB18_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB18_2 +; GFX1250-SDAG-NEXT: .LBB18_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[2:3] +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB18_5 +; GFX1250-SDAG-NEXT: .LBB18_5: +; +; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB18_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB18_4 +; GFX1250-GISEL-NEXT: .LBB18_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB18_5 +; GFX1250-GISEL-NEXT: .LBB18_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB18_2 +; GFX1250-GISEL-NEXT: .LBB18_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[4:5] +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB18_5 +; GFX1250-GISEL-NEXT: .LBB18_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw add ptr %gep0, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB19_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB19_4 +; GFX1250-SDAG-NEXT: .LBB19_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB19_5 +; GFX1250-SDAG-NEXT: .LBB19_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB19_2 +; GFX1250-SDAG-NEXT: .LBB19_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[2:3] +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB19_5 +; GFX1250-SDAG-NEXT: .LBB19_5: +; +; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB19_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB19_4 +; GFX1250-GISEL-NEXT: .LBB19_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB19_5 +; GFX1250-GISEL-NEXT: .LBB19_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB19_2 +; GFX1250-GISEL-NEXT: .LBB19_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[4:5] +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB19_5 +; GFX1250-GISEL-NEXT: .LBB19_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw add ptr %gep1, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB20_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB20_4 +; GFX1250-SDAG-NEXT: .LBB20_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB20_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB20_2 +; GFX1250-SDAG-NEXT: .LBB20_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3] +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB20_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB20_4 +; GFX1250-GISEL-NEXT: .LBB20_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB20_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB20_2 +; GFX1250-GISEL-NEXT: .LBB20_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5] +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw add ptr %gep0, i64 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB21_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB21_4 +; GFX1250-SDAG-NEXT: .LBB21_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB21_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB21_2 +; GFX1250-SDAG-NEXT: .LBB21_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3] +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB21_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB21_4 +; GFX1250-GISEL-NEXT: .LBB21_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB21_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB21_2 +; GFX1250-GISEL-NEXT: .LBB21_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[4:5] +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw add ptr %gep1, i64 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw sub +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_sub_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw sub ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_sub_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw sub ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_sub_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw sub ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_sub_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw sub ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB26_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB26_4 +; GFX1250-SDAG-NEXT: .LBB26_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB26_5 +; GFX1250-SDAG-NEXT: .LBB26_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB26_2 +; GFX1250-SDAG-NEXT: .LBB26_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_sub_co_u32 v2, vcc_lo, v0, v2 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_sub_co_ci_u32_e64 v3, null, v1, v3, vcc_lo +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB26_5 +; GFX1250-SDAG-NEXT: .LBB26_5: +; +; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB26_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB26_4 +; GFX1250-GISEL-NEXT: .LBB26_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB26_5 +; GFX1250-GISEL-NEXT: .LBB26_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB26_2 +; GFX1250-GISEL-NEXT: .LBB26_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v0, v4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_sub_co_ci_u32_e64 v3, null, v1, v5, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB26_5 +; GFX1250-GISEL-NEXT: .LBB26_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw sub ptr %gep0, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB27_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB27_4 +; GFX1250-SDAG-NEXT: .LBB27_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB27_5 +; GFX1250-SDAG-NEXT: .LBB27_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB27_2 +; GFX1250-SDAG-NEXT: .LBB27_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_sub_co_u32 v2, vcc_lo, v0, v2 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_sub_co_ci_u32_e64 v3, null, v1, v3, vcc_lo +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB27_5 +; GFX1250-SDAG-NEXT: .LBB27_5: +; +; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB27_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB27_4 +; GFX1250-GISEL-NEXT: .LBB27_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB27_5 +; GFX1250-GISEL-NEXT: .LBB27_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB27_2 +; GFX1250-GISEL-NEXT: .LBB27_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_sub_co_u32 v2, vcc_lo, v0, v4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_sub_co_ci_u32_e64 v3, null, v1, v5, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB27_5 +; GFX1250-GISEL-NEXT: .LBB27_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw sub ptr %gep1, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB28_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB28_4 +; GFX1250-SDAG-NEXT: .LBB28_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB28_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB28_2 +; GFX1250-SDAG-NEXT: .LBB28_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v2 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB28_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB28_4 +; GFX1250-GISEL-NEXT: .LBB28_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB28_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB28_2 +; GFX1250-GISEL-NEXT: .LBB28_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v5, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw sub ptr %gep0, i64 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB29_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB29_4 +; GFX1250-SDAG-NEXT: .LBB29_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB29_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB29_2 +; GFX1250-SDAG-NEXT: .LBB29_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v2 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB29_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB29_4 +; GFX1250-GISEL-NEXT: .LBB29_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB29_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB29_2 +; GFX1250-GISEL-NEXT: .LBB29_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v5, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw sub ptr %gep1, i64 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw and +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_and_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw and ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_and_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw and ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_and_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw and ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_and_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw and ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB34_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB34_4 +; GFX1250-SDAG-NEXT: .LBB34_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB34_5 +; GFX1250-SDAG-NEXT: .LBB34_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB34_2 +; GFX1250-SDAG-NEXT: .LBB34_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v3, v1, v3 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB34_5 +; GFX1250-SDAG-NEXT: .LBB34_5: +; +; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB34_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB34_4 +; GFX1250-GISEL-NEXT: .LBB34_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB34_5 +; GFX1250-GISEL-NEXT: .LBB34_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB34_2 +; GFX1250-GISEL-NEXT: .LBB34_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, v0, v4 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v3, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB34_5 +; GFX1250-GISEL-NEXT: .LBB34_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB35_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB35_4 +; GFX1250-SDAG-NEXT: .LBB35_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB35_5 +; GFX1250-SDAG-NEXT: .LBB35_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB35_2 +; GFX1250-SDAG-NEXT: .LBB35_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v3, v1, v3 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB35_5 +; GFX1250-SDAG-NEXT: .LBB35_5: +; +; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB35_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB35_4 +; GFX1250-GISEL-NEXT: .LBB35_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB35_5 +; GFX1250-GISEL-NEXT: .LBB35_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB35_2 +; GFX1250-GISEL-NEXT: .LBB35_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, v0, v4 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v3, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB35_5 +; GFX1250-GISEL-NEXT: .LBB35_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB36_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB36_4 +; GFX1250-SDAG-NEXT: .LBB36_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB36_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB36_2 +; GFX1250-SDAG-NEXT: .LBB36_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, v1, v3 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_4 +; GFX1250-GISEL-NEXT: .LBB36_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB36_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB36_2 +; GFX1250-GISEL-NEXT: .LBB36_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v4 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB37_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB37_4 +; GFX1250-SDAG-NEXT: .LBB37_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB37_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB37_2 +; GFX1250-SDAG-NEXT: .LBB37_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, v1, v3 +; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_4 +; GFX1250-GISEL-NEXT: .LBB37_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB37_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB37_2 +; GFX1250-GISEL-NEXT: .LBB37_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v4 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw or +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_or_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_or_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB42_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB42_4 +; GFX1250-SDAG-NEXT: .LBB42_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB42_5 +; GFX1250-SDAG-NEXT: .LBB42_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB42_2 +; GFX1250-SDAG-NEXT: .LBB42_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v3, v1, v3 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v2, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB42_5 +; GFX1250-SDAG-NEXT: .LBB42_5: +; +; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB42_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB42_4 +; GFX1250-GISEL-NEXT: .LBB42_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB42_5 +; GFX1250-GISEL-NEXT: .LBB42_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB42_2 +; GFX1250-GISEL-NEXT: .LBB42_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v2, v0, v4 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB42_5 +; GFX1250-GISEL-NEXT: .LBB42_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB43_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB43_4 +; GFX1250-SDAG-NEXT: .LBB43_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB43_5 +; GFX1250-SDAG-NEXT: .LBB43_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB43_2 +; GFX1250-SDAG-NEXT: .LBB43_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v3, v1, v3 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v2, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB43_5 +; GFX1250-SDAG-NEXT: .LBB43_5: +; +; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB43_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB43_4 +; GFX1250-GISEL-NEXT: .LBB43_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB43_5 +; GFX1250-GISEL-NEXT: .LBB43_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB43_2 +; GFX1250-GISEL-NEXT: .LBB43_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v2, v0, v4 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB43_5 +; GFX1250-GISEL-NEXT: .LBB43_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB44_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB44_4 +; GFX1250-SDAG-NEXT: .LBB44_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB44_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB44_2 +; GFX1250-SDAG-NEXT: .LBB44_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_4 +; GFX1250-GISEL-NEXT: .LBB44_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB44_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB44_2 +; GFX1250-GISEL-NEXT: .LBB44_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB45_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB45_4 +; GFX1250-SDAG-NEXT: .LBB45_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB45_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB45_2 +; GFX1250-SDAG-NEXT: .LBB45_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v1, v1, v3 +; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_4 +; GFX1250-GISEL-NEXT: .LBB45_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB45_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB45_2 +; GFX1250-GISEL-NEXT: .LBB45_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v4 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw xor +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xor_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xor_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB50_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB50_4 +; GFX1250-SDAG-NEXT: .LBB50_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB50_5 +; GFX1250-SDAG-NEXT: .LBB50_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB50_2 +; GFX1250-SDAG-NEXT: .LBB50_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, v1, v3 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB50_5 +; GFX1250-SDAG-NEXT: .LBB50_5: +; +; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB50_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB50_4 +; GFX1250-GISEL-NEXT: .LBB50_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB50_5 +; GFX1250-GISEL-NEXT: .LBB50_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB50_2 +; GFX1250-GISEL-NEXT: .LBB50_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, v0, v4 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v3, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB50_5 +; GFX1250-GISEL-NEXT: .LBB50_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB51_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB51_4 +; GFX1250-SDAG-NEXT: .LBB51_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB51_5 +; GFX1250-SDAG-NEXT: .LBB51_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB51_2 +; GFX1250-SDAG-NEXT: .LBB51_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, v1, v3 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB51_5 +; GFX1250-SDAG-NEXT: .LBB51_5: +; +; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB51_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB51_4 +; GFX1250-GISEL-NEXT: .LBB51_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB51_5 +; GFX1250-GISEL-NEXT: .LBB51_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB51_2 +; GFX1250-GISEL-NEXT: .LBB51_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, v0, v4 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v3, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB51_5 +; GFX1250-GISEL-NEXT: .LBB51_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw xor ptr %gep1, i64 %data syncscope("agent") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB52_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB52_4 +; GFX1250-SDAG-NEXT: .LBB52_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB52_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB52_2 +; GFX1250-SDAG-NEXT: .LBB52_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_4 +; GFX1250-GISEL-NEXT: .LBB52_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB52_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB52_2 +; GFX1250-GISEL-NEXT: .LBB52_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst + ret void +} + +define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB53_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB53_4 +; GFX1250-SDAG-NEXT: .LBB53_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB53_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB53_2 +; GFX1250-SDAG-NEXT: .LBB53_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, v1, v3 +; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_4 +; GFX1250-GISEL-NEXT: .LBB53_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB53_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB53_2 +; GFX1250-GISEL-NEXT: .LBB53_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4 +; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw xor ptr %gep1, i64 %data syncscope("agent") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw max +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_max_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw max ptr %gep0, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_max_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw max ptr %gep1, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_max_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw max ptr %gep0, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_max_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw max ptr %gep1, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB58_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB58_4 +; GFX1250-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB58_5 +; GFX1250-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB58_2 +; GFX1250-SDAG-NEXT: .LBB58_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB58_5 +; GFX1250-SDAG-NEXT: .LBB58_5: +; +; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB58_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB58_4 +; GFX1250-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB58_5 +; GFX1250-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB58_2 +; GFX1250-GISEL-NEXT: .LBB58_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB58_5 +; GFX1250-GISEL-NEXT: .LBB58_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw max ptr %gep0, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB59_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB59_4 +; GFX1250-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB59_5 +; GFX1250-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB59_2 +; GFX1250-SDAG-NEXT: .LBB59_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB59_5 +; GFX1250-SDAG-NEXT: .LBB59_5: +; +; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB59_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB59_4 +; GFX1250-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB59_5 +; GFX1250-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB59_2 +; GFX1250-GISEL-NEXT: .LBB59_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB59_5 +; GFX1250-GISEL-NEXT: .LBB59_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw max ptr %gep1, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB60_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB60_4 +; GFX1250-SDAG-NEXT: .LBB60_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB60_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB60_2 +; GFX1250-SDAG-NEXT: .LBB60_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB60_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB60_4 +; GFX1250-GISEL-NEXT: .LBB60_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB60_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB60_2 +; GFX1250-GISEL-NEXT: .LBB60_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw max ptr %gep0, i64 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB61_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB61_4 +; GFX1250-SDAG-NEXT: .LBB61_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB61_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB61_2 +; GFX1250-SDAG-NEXT: .LBB61_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB61_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB61_4 +; GFX1250-GISEL-NEXT: .LBB61_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB61_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB61_2 +; GFX1250-GISEL-NEXT: .LBB61_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw max ptr %gep1, i64 %data syncscope("workgroup") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw min +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_min_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw min ptr %gep0, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_min_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw min ptr %gep1, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_min_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw min ptr %gep0, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_min_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw min ptr %gep1, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB66_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB66_4 +; GFX1250-SDAG-NEXT: .LBB66_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB66_5 +; GFX1250-SDAG-NEXT: .LBB66_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB66_2 +; GFX1250-SDAG-NEXT: .LBB66_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB66_5 +; GFX1250-SDAG-NEXT: .LBB66_5: +; +; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB66_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB66_4 +; GFX1250-GISEL-NEXT: .LBB66_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB66_5 +; GFX1250-GISEL-NEXT: .LBB66_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB66_2 +; GFX1250-GISEL-NEXT: .LBB66_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB66_5 +; GFX1250-GISEL-NEXT: .LBB66_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw min ptr %gep0, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB67_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB67_4 +; GFX1250-SDAG-NEXT: .LBB67_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB67_5 +; GFX1250-SDAG-NEXT: .LBB67_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB67_2 +; GFX1250-SDAG-NEXT: .LBB67_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB67_5 +; GFX1250-SDAG-NEXT: .LBB67_5: +; +; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB67_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB67_4 +; GFX1250-GISEL-NEXT: .LBB67_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB67_5 +; GFX1250-GISEL-NEXT: .LBB67_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB67_2 +; GFX1250-GISEL-NEXT: .LBB67_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB67_5 +; GFX1250-GISEL-NEXT: .LBB67_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw min ptr %gep1, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB68_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB68_4 +; GFX1250-SDAG-NEXT: .LBB68_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB68_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB68_2 +; GFX1250-SDAG-NEXT: .LBB68_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB68_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB68_4 +; GFX1250-GISEL-NEXT: .LBB68_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB68_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB68_2 +; GFX1250-GISEL-NEXT: .LBB68_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw min ptr %gep0, i64 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB69_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB69_4 +; GFX1250-SDAG-NEXT: .LBB69_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB69_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB69_2 +; GFX1250-SDAG-NEXT: .LBB69_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_i64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB69_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB69_4 +; GFX1250-GISEL-NEXT: .LBB69_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB69_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB69_2 +; GFX1250-GISEL-NEXT: .LBB69_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw min ptr %gep1, i64 %data syncscope("workgroup") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw umax +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umax_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw umax ptr %gep0, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umax_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw umax ptr %gep1, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umax_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw umax ptr %gep0, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umax_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw umax ptr %gep1, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB74_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB74_4 +; GFX1250-SDAG-NEXT: .LBB74_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB74_5 +; GFX1250-SDAG-NEXT: .LBB74_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB74_2 +; GFX1250-SDAG-NEXT: .LBB74_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB74_5 +; GFX1250-SDAG-NEXT: .LBB74_5: +; +; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB74_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB74_4 +; GFX1250-GISEL-NEXT: .LBB74_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB74_5 +; GFX1250-GISEL-NEXT: .LBB74_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB74_2 +; GFX1250-GISEL-NEXT: .LBB74_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB74_5 +; GFX1250-GISEL-NEXT: .LBB74_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw umax ptr %gep0, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB75_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB75_4 +; GFX1250-SDAG-NEXT: .LBB75_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB75_5 +; GFX1250-SDAG-NEXT: .LBB75_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB75_2 +; GFX1250-SDAG-NEXT: .LBB75_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB75_5 +; GFX1250-SDAG-NEXT: .LBB75_5: +; +; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB75_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB75_4 +; GFX1250-GISEL-NEXT: .LBB75_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB75_5 +; GFX1250-GISEL-NEXT: .LBB75_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB75_2 +; GFX1250-GISEL-NEXT: .LBB75_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB75_5 +; GFX1250-GISEL-NEXT: .LBB75_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw umax ptr %gep1, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB76_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB76_4 +; GFX1250-SDAG-NEXT: .LBB76_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB76_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB76_2 +; GFX1250-SDAG-NEXT: .LBB76_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB76_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB76_4 +; GFX1250-GISEL-NEXT: .LBB76_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB76_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB76_2 +; GFX1250-GISEL-NEXT: .LBB76_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw umax ptr %gep0, i64 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB77_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB77_4 +; GFX1250-SDAG-NEXT: .LBB77_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB77_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB77_2 +; GFX1250-SDAG-NEXT: .LBB77_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB77_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB77_4 +; GFX1250-GISEL-NEXT: .LBB77_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB77_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB77_2 +; GFX1250-GISEL-NEXT: .LBB77_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw umax ptr %gep1, i64 %data syncscope("workgroup") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; atomicrmw umin +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umin_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw umin ptr %gep0, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umin_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw umin ptr %gep1, i32 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umin_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw umin ptr %gep0, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_umin_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_dscnt 0x0 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw umin ptr %gep1, i32 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB82_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB82_4 +; GFX1250-SDAG-NEXT: .LBB82_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB82_5 +; GFX1250-SDAG-NEXT: .LBB82_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB82_2 +; GFX1250-SDAG-NEXT: .LBB82_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB82_5 +; GFX1250-SDAG-NEXT: .LBB82_5: +; +; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB82_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB82_4 +; GFX1250-GISEL-NEXT: .LBB82_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB82_5 +; GFX1250-GISEL-NEXT: .LBB82_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB82_2 +; GFX1250-GISEL-NEXT: .LBB82_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB82_5 +; GFX1250-GISEL-NEXT: .LBB82_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw umin ptr %gep0, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB83_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB83_4 +; GFX1250-SDAG-NEXT: .LBB83_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB83_5 +; GFX1250-SDAG-NEXT: .LBB83_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB83_2 +; GFX1250-SDAG-NEXT: .LBB83_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v3, v1 :: v_dual_cndmask_b32 v2, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB83_5 +; GFX1250-SDAG-NEXT: .LBB83_5: +; +; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB83_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB83_4 +; GFX1250-GISEL-NEXT: .LBB83_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB83_5 +; GFX1250-GISEL-NEXT: .LBB83_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB83_2 +; GFX1250-GISEL-NEXT: .LBB83_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v4, v0 :: v_dual_cndmask_b32 v3, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB83_5 +; GFX1250-GISEL-NEXT: .LBB83_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw umin ptr %gep1, i64 %data syncscope("workgroup") seq_cst + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB84_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB84_4 +; GFX1250-SDAG-NEXT: .LBB84_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB84_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB84_2 +; GFX1250-SDAG-NEXT: .LBB84_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB84_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB84_4 +; GFX1250-GISEL-NEXT: .LBB84_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB84_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB84_2 +; GFX1250-GISEL-NEXT: .LBB84_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw umin ptr %gep0, i64 %data syncscope("workgroup") seq_cst + ret void +} + +define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB85_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB85_4 +; GFX1250-SDAG-NEXT: .LBB85_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB85_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB85_2 +; GFX1250-SDAG-NEXT: .LBB85_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_le_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v3, v1 :: v_dual_cndmask_b32 v0, v2, v0 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB85_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB85_4 +; GFX1250-GISEL-NEXT: .LBB85_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB85_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB85_2 +; GFX1250-GISEL-NEXT: .LBB85_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw umin ptr %gep1, i64 %data syncscope("workgroup") seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; cmpxchg +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) { +; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v3, v1 +; GFX1250-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt 0x0 +; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %cmpxchg = cmpxchg ptr %gep0, i32 %cmp, i32 %data seq_cst seq_cst + %rtn = extractvalue { i32, i1 } %cmpxchg, 0 + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) { +; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v3, v1 +; GFX1250-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt 0x0 +; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %cmpxchg = cmpxchg ptr %gep1, i32 %cmp, i32 %data seq_cst seq_cst + %rtn = extractvalue { i32, i1 } %cmpxchg, 0 + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) { +; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v3, v1 +; GFX1250-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt 0x0 +; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v[2:3], s[2:3] scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = cmpxchg ptr %gep0, i32 %cmp, i32 %data seq_cst seq_cst + ret void +} + +define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) { +; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v3, v1 +; GFX1250-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt 0x0 +; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v[2:3], s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = cmpxchg ptr %gep1, i32 %cmp, i32 %data seq_cst seq_cst + ret void +} + +define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) { +; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v4 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v4, v3 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB90_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB90_4 +; GFX1250-SDAG-NEXT: .LBB90_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB90_5 +; GFX1250-SDAG-NEXT: .LBB90_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0 +; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[2:3], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB90_2 +; GFX1250-SDAG-NEXT: .LBB90_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v8, -1, v2, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v8, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v1, v5 :: v_dual_cndmask_b32 v2, v0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v8, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB90_5 +; GFX1250-SDAG-NEXT: .LBB90_5: +; +; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v7, v4 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v0, v5 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB90_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB90_4 +; GFX1250-GISEL-NEXT: .LBB90_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB90_5 +; GFX1250-GISEL-NEXT: .LBB90_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0 +; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v5, v[6:9], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB90_2 +; GFX1250-GISEL-NEXT: .LBB90_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v0, v6 :: v_dual_cndmask_b32 v3, v1, v7 +; GFX1250-GISEL-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB90_5 +; GFX1250-GISEL-NEXT: .LBB90_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %cmpxchg = cmpxchg ptr %gep0, i64 %cmp, i64 %data seq_cst seq_cst + %rtn = extractvalue { i64, i1 } %cmpxchg, 0 + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) { +; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB91_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB91_4 +; GFX1250-SDAG-NEXT: .LBB91_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB91_5 +; GFX1250-SDAG-NEXT: .LBB91_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0 +; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[2:3], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB91_2 +; GFX1250-SDAG-NEXT: .LBB91_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v8, -1, v2, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v8, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v1, v5 :: v_dual_cndmask_b32 v2, v0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v8, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB91_5 +; GFX1250-SDAG-NEXT: .LBB91_5: +; +; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v7, v4 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v5 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB91_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB91_4 +; GFX1250-GISEL-NEXT: .LBB91_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB91_5 +; GFX1250-GISEL-NEXT: .LBB91_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0 +; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v5, v[6:9], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB91_2 +; GFX1250-GISEL-NEXT: .LBB91_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v0, v6 :: v_dual_cndmask_b32 v3, v1, v7 +; GFX1250-GISEL-NEXT: scratch_store_b64 v4, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB91_5 +; GFX1250-GISEL-NEXT: .LBB91_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %cmpxchg = cmpxchg ptr %gep1, i64 %cmp, i64 %data seq_cst seq_cst + %rtn = extractvalue { i64, i1 } %cmpxchg, 0 + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) { +; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v4 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v4, v3 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB92_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB92_4 +; GFX1250-SDAG-NEXT: .LBB92_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB92_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0 +; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:7] scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB92_2 +; GFX1250-SDAG-NEXT: .LBB92_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v5 :: v_dual_cndmask_b32 v0, v0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB92_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB92_4 +; GFX1250-GISEL-NEXT: .LBB92_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB92_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0 +; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v0, v[6:9], s[2:3] scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB92_2 +; GFX1250-GISEL-NEXT: .LBB92_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v6 :: v_dual_cndmask_b32 v1, v1, v7 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = cmpxchg ptr %gep0, i64 %cmp, i64 %data seq_cst seq_cst + ret void +} + +define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) { +; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB93_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB93_4 +; GFX1250-SDAG-NEXT: .LBB93_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB93_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0 +; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:7] scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB93_2 +; GFX1250-SDAG-NEXT: .LBB93_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7] +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v5 :: v_dual_cndmask_b32 v0, v0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB93_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB93_4 +; GFX1250-GISEL-NEXT: .LBB93_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB93_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0 +; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v0, v[6:9], s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr9 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB93_2 +; GFX1250-GISEL-NEXT: .LBB93_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v6 :: v_dual_cndmask_b32 v1, v1, v7 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = cmpxchg ptr %gep1, i64 %cmp, i64 %data seq_cst seq_cst + ret void +} + +; -------------------------------------------------------------------------------- +; amdgcn atomic inc +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_inc_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw uinc_wrap ptr %gep0, i32 %data syncscope("agent") monotonic + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_inc_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw uinc_wrap ptr %gep1, i32 %data syncscope("agent") monotonic + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_inc_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw uinc_wrap ptr %gep0, i32 %data syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_inc_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw uinc_wrap ptr %gep1, i32 %data syncscope("agent") monotonic + ret void +} + +define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB98_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB98_4 +; GFX1250-SDAG-NEXT: .LBB98_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB98_5 +; GFX1250-SDAG-NEXT: .LBB98_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB98_2 +; GFX1250-SDAG-NEXT: .LBB98_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, 0, v5 :: v_dual_cndmask_b32 v2, 0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB98_5 +; GFX1250-SDAG-NEXT: .LBB98_5: +; +; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB98_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB98_4 +; GFX1250-GISEL-NEXT: .LBB98_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB98_5 +; GFX1250-GISEL-NEXT: .LBB98_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB98_2 +; GFX1250-GISEL-NEXT: .LBB98_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, 1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB98_5 +; GFX1250-GISEL-NEXT: .LBB98_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw uinc_wrap ptr %gep0, i64 %data syncscope("agent") monotonic + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB99_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB99_4 +; GFX1250-SDAG-NEXT: .LBB99_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB99_5 +; GFX1250-SDAG-NEXT: .LBB99_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB99_2 +; GFX1250-SDAG-NEXT: .LBB99_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, 0, v5 :: v_dual_cndmask_b32 v2, 0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_branch .LBB99_5 +; GFX1250-SDAG-NEXT: .LBB99_5: +; +; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB99_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB99_4 +; GFX1250-GISEL-NEXT: .LBB99_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB99_5 +; GFX1250-GISEL-NEXT: .LBB99_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB99_2 +; GFX1250-GISEL-NEXT: .LBB99_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, 1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_branch .LBB99_5 +; GFX1250-GISEL-NEXT: .LBB99_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw uinc_wrap ptr %gep1, i64 %data syncscope("agent") monotonic + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB100_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB100_4 +; GFX1250-SDAG-NEXT: .LBB100_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB100_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB100_2 +; GFX1250-SDAG-NEXT: .LBB100_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, 0, v5 :: v_dual_cndmask_b32 v0, 0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB100_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB100_4 +; GFX1250-GISEL-NEXT: .LBB100_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB100_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB100_2 +; GFX1250-GISEL-NEXT: .LBB100_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, 1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw uinc_wrap ptr %gep0, i64 %data syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB101_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB101_4 +; GFX1250-SDAG-NEXT: .LBB101_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB101_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB101_2 +; GFX1250-SDAG-NEXT: .LBB101_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffd +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, 0, v5 :: v_dual_cndmask_b32 v0, 0, v4 +; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB101_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB101_4 +; GFX1250-GISEL-NEXT: .LBB101_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB101_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB101_2 +; GFX1250-GISEL-NEXT: .LBB101_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, 1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo +; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0, vcc_lo +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw uinc_wrap ptr %gep1, i64 %data syncscope("agent") monotonic + ret void +} + +; -------------------------------------------------------------------------------- +; amdgcn atomic dec +; -------------------------------------------------------------------------------- + + +define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_dec_saddr_i32_rtn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw udec_wrap ptr %gep0, i32 %data syncscope("agent") monotonic + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_dec_saddr_i32_rtn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw udec_wrap ptr %gep1, i32 %data syncscope("agent") monotonic + %cast.rtn = bitcast i32 %rtn to float + ret float %cast.rtn +} + +define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_dec_saddr_i32_nortn: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw udec_wrap ptr %gep0, i32 %data syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_dec_saddr_i32_nortn_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw udec_wrap ptr %gep1, i32 %data syncscope("agent") monotonic + ret void +} + +define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB106_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB106_4 +; GFX1250-SDAG-NEXT: .LBB106_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB106_5 +; GFX1250-SDAG-NEXT: .LBB106_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB106_2 +; GFX1250-SDAG-NEXT: .LBB106_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -1 +; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v5, v3 :: v_dual_cndmask_b32 v2, v4, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-SDAG-NEXT: s_branch .LBB106_5 +; GFX1250-SDAG-NEXT: .LBB106_5: +; +; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB106_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB106_4 +; GFX1250-GISEL-NEXT: .LBB106_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB106_5 +; GFX1250-GISEL-NEXT: .LBB106_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB106_2 +; GFX1250-GISEL-NEXT: .LBB106_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, -1 +; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-GISEL-NEXT: s_branch .LBB106_5 +; GFX1250-GISEL-NEXT: .LBB106_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %rtn = atomicrmw udec_wrap ptr %gep0, i64 %data syncscope("agent") monotonic + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v5 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB107_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB107_4 +; GFX1250-SDAG-NEXT: .LBB107_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: s_branch .LBB107_5 +; GFX1250-SDAG-NEXT: .LBB107_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB107_2 +; GFX1250-SDAG-NEXT: .LBB107_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -1 +; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v5, v3 :: v_dual_cndmask_b32 v2, v4, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-SDAG-NEXT: s_branch .LBB107_5 +; GFX1250-SDAG-NEXT: .LBB107_5: +; +; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v5, v2 +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v7 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB107_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB107_4 +; GFX1250-GISEL-NEXT: .LBB107_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_branch .LBB107_5 +; GFX1250-GISEL-NEXT: .LBB107_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6_vgpr7 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB107_2 +; GFX1250-GISEL-NEXT: .LBB107_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc_lo +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, -1 +; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s1 +; GFX1250-GISEL-NEXT: s_branch .LBB107_5 +; GFX1250-GISEL-NEXT: .LBB107_5: + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %rtn = atomicrmw udec_wrap ptr %gep1, i64 %data syncscope("agent") monotonic + %cast.rtn = bitcast i64 %rtn to <2 x float> + ret <2 x float> %cast.rtn +} + +define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB108_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB108_4 +; GFX1250-SDAG-NEXT: .LBB108_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB108_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB108_2 +; GFX1250-SDAG-NEXT: .LBB108_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, -1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB108_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB108_4 +; GFX1250-GISEL-NEXT: .LBB108_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB108_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB108_2 +; GFX1250-GISEL-NEXT: .LBB108_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, -1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v4 :: v_dual_cndmask_b32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %unused = atomicrmw udec_wrap ptr %gep0, i64 %data syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], lit64(0xffffffffffffff80) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_cmpx_ne_u32_e64 s1, v1 +; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB109_3 +; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB109_4 +; GFX1250-SDAG-NEXT: .LBB109_2: ; %atomicrmw.phi +; GFX1250-SDAG-NEXT: s_endpgm +; GFX1250-SDAG-NEXT: .LBB109_3: ; %atomicrmw.global +; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB109_2 +; GFX1250-SDAG-NEXT: .LBB109_4: ; %atomicrmw.private +; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo +; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3] +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, -1 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-SDAG-NEXT: s_wait_alu 0xfffe +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v0, v0, v2 +; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base +; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_cmpx_ne_u32_e64 s1, v3 +; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB109_3 +; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB109_4 +; GFX1250-GISEL-NEXT: .LBB109_2: ; %atomicrmw.phi +; GFX1250-GISEL-NEXT: s_endpgm +; GFX1250-GISEL-NEXT: .LBB109_3: ; %atomicrmw.global +; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB109_2 +; GFX1250-GISEL-NEXT: .LBB109_4: ; %atomicrmw.private +; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3] +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo +; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1] +; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, -1 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffe +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v4 :: v_dual_cndmask_b32 v1, v1, v5 +; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %unused = atomicrmw udec_wrap ptr %gep1, i64 %data syncscope("agent") monotonic + ret void +} + +attributes #0 = { argmemonly nounwind willreturn } diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll new file mode 100644 index 0000000..f54fbba --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll @@ -0,0 +1,2223 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s + +; Test using saddr addressing mode of flat_*load_* instructions. + +; -------------------------------------------------------------------------------- +; No vgpr offset, constants +; -------------------------------------------------------------------------------- + +; SGPR base only +define amdgpu_ps float @flat_load_saddr_i8_offset_0(ptr inreg %sbase) { +; GFX1250-LABEL: flat_load_saddr_i8_offset_0: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %load = load i8, ptr %sbase + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; SGPR base with maximum gfx1250 immediate offset +define amdgpu_ps float @flat_load_saddr_i8_offset_8388607(ptr inreg %sbase) { +; GFX1250-LABEL: flat_load_saddr_i8_offset_8388607: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 8388607 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; SGPR base with maximum gfx1250 immediate offset + 1 +define amdgpu_ps float @flat_load_saddr_i8_offset_8388608(ptr inreg %sbase) { +; GFX1250-LABEL: flat_load_saddr_i8_offset_8388608: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 8388608 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; SGPR base with maximum negative gfx1250 immediate offset +define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388608(ptr inreg %sbase) { +; GFX1250-LABEL: flat_load_saddr_i8_offset_neg8388608: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388608 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 -8388608 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; SGPR base with maximum negative gfx1250 immediate offset -1 +define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388609(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg8388609: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0xff800000, s2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, -1, s3, s0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:-1 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg8388609: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0xff7fffff +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 -8388609 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_0xFFFFFFFF(ptr inreg %sbase) { +; GFX1250-LABEL: flat_load_saddr_i8_offset_0xFFFFFFFF: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, 0xff800000 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 4294967295 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000000(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100000000: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-SDAG-NEXT: s_add_co_i32 s3, s3, 1 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100000000: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0 +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 4294967296 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000001(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100000001: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 1, s3, s0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:1 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100000001: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 1 +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 4294967297 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000FFF(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100000FFF: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 1, s3, s0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:4095 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100000FFF: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0xfff +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 4294971391 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_0x100001000(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100001000: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 1, s3, s0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:4096 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100001000: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0x1000 +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 4294971392 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_neg0xFFFFFFFF(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg0xFFFFFFFF: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0x800000, s2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, -1, s3, s0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:-8388607 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg0xFFFFFFFF: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 1 +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 -4294967295 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000000(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg0x100000000: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-SDAG-NEXT: s_add_co_i32 s3, s3, -1 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg0x100000000: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0 +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -1 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 -4294967296 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000001(ptr inreg %sbase) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg0x100000001: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, -1, s3, s0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:-1 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg0x100000001: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, -1 +; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -2 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 -4294967297 + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; -------------------------------------------------------------------------------- +; Basic addressing patterns +; -------------------------------------------------------------------------------- + +; Basic pattern, no immediate offset. +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Maximum positive offset on gfx1250 +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 8388607 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Maximum positive offset on gfx1250 + 1 +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %sbase, i32 %voffset) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388608: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1] +; GFX1250-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x800000, v0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388608: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x800000, v0 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 8388608 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Maximum negative offset on gfx1250 +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388608(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388608: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388608 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -8388608 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Maximum negative offset on gfx1250 - 1 +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388607(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388607: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388607 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -8388607 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 8388607 + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 %zext.offset + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; pointer addressing done in integers +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %sbase.as.int = ptrtoint ptr %sbase to i64 + %add = add i64 %sbase.as.int, %zext.offset + %dirty.gep = inttoptr i64 %add to ptr + %load = load i8, ptr %dirty.gep + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; zext forced to LHS of addressing expression +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %sbase.as.int = ptrtoint ptr %sbase to i64 + %add = add i64 %zext.offset, %sbase.as.int + %dirty.gep = inttoptr i64 %add to ptr + %load = load i8, ptr %dirty.gep + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; zext forced to LHS of addressing expression, with immediate offset +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %sbase.as.int = ptrtoint ptr %sbase to i64 + %add = add i64 %zext.offset, %sbase.as.int + %add.immoffset = add i64 %add, 128 + %dirty.gep = inttoptr i64 %add.immoffset to ptr + %load = load i8, ptr %dirty.gep + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; zext forced to LHS of addressing expression, with immediate offset in non-canonical position +define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %sbase.as.int = ptrtoint ptr %sbase to i64 + %add.immoffset = add i64 %sbase.as.int, 128 + %add = add i64 %zext.offset, %add.immoffset + %dirty.gep = inttoptr i64 %add to ptr + %load = load i8, ptr %dirty.gep + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; -------------------------------------------------------------------------------- +; Uniformity edge cases +; -------------------------------------------------------------------------------- + +@ptr.in.lds = internal addrspace(3) global ptr undef + +; Base pointer is uniform, but also in VGPRs +define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) { +; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Base pointer is uniform, but also in VGPRs, with imm offset +define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset) { +; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1] offset:42 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:42 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Both 64-bit base and 32-bit offset are scalar +define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset(ptr inreg %sbase, i32 inreg %soffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_uniform_offset: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, s4 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %soffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Both 64-bit base and 32-bit offset are scalar, with immediate offset. +define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset_immoffset(ptr inreg %sbase, i32 inreg %soffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_uniform_offset_immoffset: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, s4 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-24 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %soffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -24 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Both components uniform, zext forced to LHS of addressing expression +define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 inreg %soffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, s4 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %soffset to i64 + %sbase.as.int = ptrtoint ptr %sbase to i64 + %add = add i64 %zext.offset, %sbase.as.int + %dirty.gep = inttoptr i64 %add to ptr + %load = load i8, ptr %dirty.gep + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; Both components uniform, zext forced to LHS of addressing expression, with immediate offset +define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 inreg %soffset) { +; GFX1250-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_mov_b32_e32 v0, s4 +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %soffset to i64 + %sbase.as.int = ptrtoint ptr %sbase to i64 + %add = add i64 %zext.offset, %sbase.as.int + %add.immoffset = add i64 %add, 128 + %dirty.gep = inttoptr i64 %add.immoffset to ptr + %load = load i8, ptr %dirty.gep + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; divergent 64-bit base, 32-bit scalar offset. +define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffset) { +; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3] +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %soffset to i64 + %gep0 = getelementptr inbounds i8, ptr %vbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; divergent 64-bit base, 32-bit scalar offset, with imm offset +define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i32 inreg %soffset) { +; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3] +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:8388607 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:8388607 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %soffset to i64 + %gep0 = getelementptr inbounds i8, ptr %vbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 8388607 + %load = load i8, ptr %gep1 + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; -------------------------------------------------------------------------------- +; Natural addressing shifts with restricted range +; -------------------------------------------------------------------------------- + +; Cannot push the shift into 32-bits, and cannot match. +define amdgpu_ps float @flat_load_saddr_f32_natural_addressing(ptr inreg %sbase, ptr %voffset.ptr) { +; GFX1250-SDAG-LABEL: flat_load_saddr_f32_natural_addressing: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[2:3] +; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_f32_natural_addressing: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1] +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %voffset = load i32, ptr %voffset.ptr + %zext.offset = zext i32 %voffset to i64 + %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset + %load = load float, ptr %gep + ret float %load +} + +; Cannot push the shift into 32-bits, with an immediate offset. +define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inreg %sbase, ptr %voffset.ptr) { +; GFX1250-LABEL: flat_load_saddr_f32_natural_addressing_immoffset: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %voffset = load i32, ptr %voffset.ptr + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 128 + %load = load float, ptr %gep1 + ret float %load +} + +; Range is sufficiently restricted to push the shift into 32-bits. +define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, ptr %voffset.ptr) { +; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{} + %zext.offset = zext i32 %voffset to i64 + %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset + %load = load float, ptr %gep + ret float %load +} + +; Range is sufficiently restricted to push the shift into 32-bits, with an imm offset +define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg %sbase, ptr %voffset.ptr) { +; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range_imm_offset: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:400 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{} + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds float, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds float, ptr %gep0, i64 100 + %load = load float, ptr %gep1 + ret float %load +} + +; Range is 1 beyond the limit where we can move the shift into 32-bits. +define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_too_large(ptr inreg %sbase, ptr %voffset.ptr) { +; GFX1250-SDAG-LABEL: flat_load_f32_saddr_zext_vgpr_range_too_large: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[2:3] +; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_f32_saddr_zext_vgpr_range_too_large: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_lshlrev_b64_e32 v[0:1], 2, v[0:1] +; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v3, v1, vcc_lo +; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %voffset = load i32, ptr %voffset.ptr, !range !1, !noundef !{} + %zext.offset = zext i32 %voffset to i64 + %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset + %load = load float, ptr %gep + ret float %load +} + +; -------------------------------------------------------------------------------- +; Stress various type loads +; -------------------------------------------------------------------------------- + +define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %cast.load = bitcast i16 %load to half + ret half %cast.load +} + +define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %cast.load = bitcast i16 %load to half + ret half %cast.load +} + +define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_f16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load half, ptr %gep0 + ret half %load +} + +define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_f16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load half, ptr %gep1 + ret half %load +} + +define amdgpu_ps float @flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i32, ptr %gep0 + %cast.load = bitcast i32 %load to float + ret float %cast.load +} + +define amdgpu_ps float @flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i32, ptr %gep1 + %cast.load = bitcast i32 %load to float + ret float %cast.load +} + +define amdgpu_ps float @flat_load_saddr_f32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load float, ptr %gep0 + ret float %load +} + +define amdgpu_ps float @flat_load_saddr_f32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_f32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load float, ptr %gep1 + ret float %load +} + +define amdgpu_ps <2 x half> @flat_load_saddr_v2i16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2i16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <2 x i16>, ptr %gep0 + %cast.load = bitcast <2 x i16> %load to <2 x half> + ret <2 x half> %cast.load +} + +define amdgpu_ps <2 x half> @flat_load_saddr_v2i16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2i16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <2 x i16>, ptr %gep1 + %cast.load = bitcast <2 x i16> %load to <2 x half> + ret <2 x half> %cast.load +} + +define amdgpu_ps <2 x half> @flat_load_saddr_v2f16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2f16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <2 x half>, ptr %gep0 + ret <2 x half> %load +} + +define amdgpu_ps <2 x half> @flat_load_saddr_v2f16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2f16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <2 x half>, ptr %gep1 + ret <2 x half> %load +} + +define amdgpu_ps <2 x half> @flat_load_saddr_p3(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_p3: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load ptr addrspace(3), ptr %gep0 + %cast.load0 = ptrtoint ptr addrspace(3) %load to i32 + %cast.load1 = bitcast i32 %cast.load0 to <2 x half> + ret <2 x half> %cast.load1 +} + +define amdgpu_ps <2 x half> @flat_load_saddr_p3_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_p3_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load ptr addrspace(3), ptr %gep1 + %cast.load0 = ptrtoint ptr addrspace(3) %load to i32 + %cast.load1 = bitcast i32 %cast.load0 to <2 x half> + ret <2 x half> %cast.load1 +} + +define amdgpu_ps <2 x float> @flat_load_saddr_f64(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_f64: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load double, ptr %gep0 + %cast.load = bitcast double %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_f64_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_f64_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load double, ptr %gep1 + %cast.load = bitcast double %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i64: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i64, ptr %gep0 + %cast.load = bitcast i64 %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_i64_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i64_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i64, ptr %gep1 + %cast.load = bitcast i64 %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v2f32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <2 x float>, ptr %gep0 + ret <2 x float> %load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v2f32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2f32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <2 x float>, ptr %gep1 + ret <2 x float> %load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v2i32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2i32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <2 x i32>, ptr %gep0 + %cast.load = bitcast <2 x i32> %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v2i32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2i32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <2 x i32>, ptr %gep1 + %cast.load = bitcast <2 x i32> %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v4i16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4i16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <4 x i16>, ptr %gep0 + %cast.load = bitcast <4 x i16> %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v4i16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4i16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <4 x i16>, ptr %gep1 + %cast.load = bitcast <4 x i16> %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v4f16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4f16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <4 x half>, ptr %gep0 + %cast.load = bitcast <4 x half> %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_v4f16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4f16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <4 x half>, ptr %gep1 + %cast.load = bitcast <4 x half> %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @flat_load_saddr_p1(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_p1: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load ptr, ptr %gep0 + %cast.load0 = ptrtoint ptr %load to i64 + %cast.load1 = bitcast i64 %cast.load0 to <2 x float> + ret <2 x float> %cast.load1 +} + +define amdgpu_ps <2 x float> @flat_load_saddr_p1_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_p1_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load ptr, ptr %gep1 + %cast.load0 = ptrtoint ptr %load to i64 + %cast.load1 = bitcast i64 %cast.load0 to <2 x float> + ret <2 x float> %cast.load1 +} + +define amdgpu_ps <3 x float> @flat_load_saddr_v3f32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v3f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <3 x float>, ptr %gep0 + ret <3 x float> %load +} + +define amdgpu_ps <3 x float> @flat_load_saddr_v3f32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v3f32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <3 x float>, ptr %gep1 + ret <3 x float> %load +} + +define amdgpu_ps <3 x float> @flat_load_saddr_v3i32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v3i32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <3 x i32>, ptr %gep0 + %cast.load = bitcast <3 x i32> %load to <3 x float> + ret <3 x float> %cast.load +} + +define amdgpu_ps <3 x float> @flat_load_saddr_v3i32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v3i32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <3 x i32>, ptr %gep1 + %cast.load = bitcast <3 x i32> %load to <3 x float> + ret <3 x float> %cast.load +} + +define amdgpu_ps <6 x half> @flat_load_saddr_v6f16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v6f16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <6 x half>, ptr %gep0 + ret <6 x half> %load +} + +define amdgpu_ps <6 x half> @flat_load_saddr_v6f16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v6f16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <6 x half>, ptr %gep1 + ret <6 x half> %load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v4f32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4f32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <4 x float>, ptr %gep0 + ret <4 x float> %load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v4f32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4f32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <4 x float>, ptr %gep1 + ret <4 x float> %load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v4i32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4i32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <4 x i32>, ptr %gep0 + %cast.load = bitcast <4 x i32> %load to <4 x float> + ret <4 x float> %cast.load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v4i32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4i32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <4 x i32>, ptr %gep1 + %cast.load = bitcast <4 x i32> %load to <4 x float> + ret <4 x float> %cast.load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v2i64(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2i64: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <2 x i64>, ptr %gep0 + %cast.load = bitcast <2 x i64> %load to <4 x float> + ret <4 x float> %cast.load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v2i64_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2i64_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <2 x i64>, ptr %gep1 + %cast.load = bitcast <2 x i64> %load to <4 x float> + ret <4 x float> %cast.load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_i128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i128, ptr %gep0 + %cast.load = bitcast i128 %load to <4 x float> + ret <4 x float> %cast.load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_i128_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i128_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i128, ptr %gep1 + %cast.load = bitcast i128 %load to <4 x float> + ret <4 x float> %cast.load +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v2p1(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2p1: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <2 x ptr>, ptr %gep0 + %cast.load0 = ptrtoint <2 x ptr> %load to <2 x i64> + %cast.load1 = bitcast <2 x i64> %cast.load0 to <4 x float> + ret <4 x float> %cast.load1 +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v2p1_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v2p1_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <2 x ptr>, ptr %gep1 + %cast.load0 = ptrtoint <2 x ptr> %load to <2 x i64> + %cast.load1 = bitcast <2 x i64> %cast.load0 to <4 x float> + ret <4 x float> %cast.load1 +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v4p3(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4p3: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load <4 x ptr addrspace(3)>, ptr %gep0 + %cast.load0 = ptrtoint <4 x ptr addrspace(3)> %load to <4 x i32> + %cast.load1 = bitcast <4 x i32> %cast.load0 to <4 x float> + ret <4 x float> %cast.load1 +} + +define amdgpu_ps <4 x float> @flat_load_saddr_v4p3_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_v4p3_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load <4 x ptr addrspace(3)>, ptr %gep1 + %cast.load0 = ptrtoint <4 x ptr addrspace(3)> %load to <4 x i32> + %cast.load1 = bitcast <4 x i32> %cast.load0 to <4 x float> + ret <4 x float> %cast.load1 +} + +; -------------------------------------------------------------------------------- +; Extending loads +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @flat_sextload_saddr_i8(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_sextload_saddr_i8: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_i8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %sextload = sext i8 %load to i32 + %cast.load = bitcast i32 %sextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_sextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_sextload_saddr_i8_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i8, ptr %gep1 + %sextload = sext i8 %load to i32 + %cast.load = bitcast i32 %sextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_sextload_saddr_i16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_sextload_saddr_i16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_i16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %sextload = sext i16 %load to i32 + %cast.load = bitcast i32 %sextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_sextload_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_sextload_saddr_i16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_i16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %sextload = sext i16 %load to i32 + %cast.load = bitcast i32 %sextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_zextload_saddr_i8(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_zextload_saddr_i8: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zextload = zext i8 %load to i32 + %cast.load = bitcast i32 %zextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_zextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_zextload_saddr_i8_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i8, ptr %gep1 + %zextload = zext i8 %load to i32 + %cast.load = bitcast i32 %zextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_zextload_saddr_i16(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_zextload_saddr_i16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %zextload = zext i16 %load to i32 + %cast.load = bitcast i32 %zextload to float + ret float %cast.load +} + +define amdgpu_ps float @flat_zextload_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_zextload_saddr_i16_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %zextload = zext i16 %load to i32 + %cast.load = bitcast i32 %zextload to float + ret float %cast.load +} + +; -------------------------------------------------------------------------------- +; Atomic load +; -------------------------------------------------------------------------------- + +define amdgpu_ps float @atomic_flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: atomic_flat_load_saddr_i32: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load atomic i32, ptr %gep0 seq_cst, align 4 + %cast.load = bitcast i32 %load to float + ret float %cast.load +} + +define amdgpu_ps float @atomic_flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: atomic_flat_load_saddr_i32_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load atomic i32, ptr %gep1 seq_cst, align 4 + %cast.load = bitcast i32 %load to float + ret float %cast.load +} + +define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: atomic_flat_load_saddr_i64: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load atomic i64, ptr %gep0 seq_cst, align 8 + %cast.load = bitcast i64 %load to <2 x float> + ret <2 x float> %cast.load +} + +define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: atomic_flat_load_saddr_i64_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: global_inv scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load atomic i64, ptr %gep1 seq_cst, align 8 + %cast.load = bitcast i64 %load to <2 x float> + ret <2 x float> %cast.load +} + +; -------------------------------------------------------------------------------- +; D16 load (low 16) +; -------------------------------------------------------------------------------- + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_d16lo_undef_hi: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %build = insertelement <2 x i16> undef, i16 %load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_d16lo_undef_hi_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %build = insertelement <2 x i16> undef, i16 %load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_d16lo_zero_hi: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_d16lo_zero_hi_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_reg_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %build = insertelement <2 x i16> %reg, i16 %load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %build = insertelement <2 x i16> %reg, i16 %load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zext.load = zext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %zext.load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i8, ptr %gep1 + %zext.load = zext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %zext.load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_i8 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %sext.load = sext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %sext.load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i8, ptr %gep1 + %sext.load = sext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %sext.load, i32 0 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +; -------------------------------------------------------------------------------- +; D16 hi load (hi16) +; -------------------------------------------------------------------------------- + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_d16hi_undef_hi: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %build = insertelement <2 x i16> undef, i16 %load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-LABEL: flat_load_saddr_i16_d16hi_undef_hi_immneg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %build = insertelement <2 x i16> undef, i16 %load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase, i32 %voffset) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_zero_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, 0, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zero_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inreg %sbase, i32 %voffset) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, 0, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_reg_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i16, ptr %gep0 + %build = insertelement <2 x i16> %reg, i16 %load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i16, ptr %gep1 + %build = insertelement <2 x i16> %reg, i16 %load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %zext.load = zext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %zext.load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i8, ptr %gep1 + %zext.load = zext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %zext.load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_i8 v0, v0, s[2:3] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %load = load i8, ptr %gep0 + %sext.load = sext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %sext.load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %load = load i8, ptr %gep1 + %sext.load = sext i8 %load to i16 + %build = insertelement <2 x i16> %reg, i16 %sext.load, i32 1 + %cast = bitcast <2 x i16> %build to <2 x half> + ret <2 x half> %cast +} + +; -------------------------------------------------------------------------------- +; or-with-constant as add +; -------------------------------------------------------------------------------- + +; Check add-as-or with split 64-bit or. +define amdgpu_ps float @flat_load_saddr_i8_offset_or_i64_imm_offset_16(ptr addrspace(6) inreg %sbase, i32 %idx) { +; GFX1250-LABEL: flat_load_saddr_i8_offset_or_i64_imm_offset_16: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, 16, v0 bitop3:0x54 +; GFX1250-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: ; return to shader part epilog + %zext.idx = zext i32 %idx to i64 + %or = or i64 %zext.idx, 16 + %addr = inttoptr i64 %or to ptr + %load = load i8, ptr %addr + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +define amdgpu_ps float @flat_load_saddr_i8_offset_or_i64_imm_offset_4160(ptr addrspace(6) inreg %sbase, i32 %idx) { +; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_or_i64_imm_offset_4160: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, 0x1040, v0 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: ; return to shader part epilog +; +; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_or_i64_imm_offset_4160: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, 0x1040, v0 +; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] +; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-GISEL-NEXT: ; return to shader part epilog + %zext.idx = zext i32 %idx to i64 + %or = or i64 %zext.idx, 4160 + %addr = inttoptr i64 %or to ptr + %load = load i8, ptr %addr + %zext = zext i8 %load to i32 + %to.vgpr = bitcast i32 %zext to float + ret float %to.vgpr +} + +; -------------------------------------------------------------------------------- +; Full 64-bit scalar add. +; -------------------------------------------------------------------------------- + +define amdgpu_ps void @flat_addr_64bit_lsr_iv(ptr inreg %arg) { +; GFX1250-SDAG-LABEL: flat_addr_64bit_lsr_iv: +; GFX1250-SDAG: ; %bb.0: ; %bb +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0 +; GFX1250-SDAG-NEXT: .LBB116_1: ; %bb3 +; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_add_nc_u64 s[4:5], s[2:3], s[0:1] +; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: flat_load_b32 v1, v0, s[4:5] scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, 0x400 +; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB116_1 +; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_addr_64bit_lsr_iv: +; GFX1250-GISEL: ; %bb.0: ; %bb +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], 0 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] +; GFX1250-GISEL-NEXT: .LBB116_1: ; %bb3 +; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, 4 +; GFX1250-GISEL-NEXT: flat_load_b32 v4, v[4:5] scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x400, v2 +; GFX1250-GISEL-NEXT: s_cbranch_vccz .LBB116_1 +; GFX1250-GISEL-NEXT: ; %bb.2: ; %bb2 +; GFX1250-GISEL-NEXT: s_endpgm +bb: + br label %bb3 + +bb2: ; preds = %bb3 + ret void + +bb3: ; preds = %bb3, %bb + %i = phi i32 [ 0, %bb ], [ %i8, %bb3 ] + %i4 = zext i32 %i to i64 + %i5 = getelementptr inbounds float, ptr %arg, i64 %i4 + %i6 = load volatile float, ptr %i5, align 4 + %i8 = add nuw nsw i32 %i, 1 + %i9 = icmp eq i32 %i8, 256 + br i1 %i9, label %bb2, label %bb3 +} + +; Make sure we only have a single zero vaddr initialization. + +define amdgpu_ps void @flat_addr_64bit_lsr_iv_multiload(ptr inreg %arg, ptr inreg %arg.1) { +; GFX1250-SDAG-LABEL: flat_addr_64bit_lsr_iv_multiload: +; GFX1250-SDAG: ; %bb.0: ; %bb +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0 +; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0 +; GFX1250-SDAG-NEXT: .LBB117_1: ; %bb3 +; GFX1250-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 +; GFX1250-SDAG-NEXT: s_add_nc_u64 s[4:5], s[2:3], s[0:1] +; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: flat_load_b32 v1, v0, s[4:5] scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-NEXT: flat_load_b32 v1, v0, s[4:5] scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, 0x400 +; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB117_1 +; GFX1250-SDAG-NEXT: ; %bb.2: ; %bb2 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_addr_64bit_lsr_iv_multiload: +; GFX1250-GISEL: ; %bb.0: ; %bb +; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], 0 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] +; GFX1250-GISEL-NEXT: .LBB117_1: ; %bb3 +; GFX1250-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2 +; GFX1250-GISEL-NEXT: s_wait_alu 0xfffd +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, 4 +; GFX1250-GISEL-NEXT: flat_load_b32 v6, v[4:5] scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: flat_load_b32 v4, v[4:5] scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x400, v2 +; GFX1250-GISEL-NEXT: s_cbranch_vccz .LBB117_1 +; GFX1250-GISEL-NEXT: ; %bb.2: ; %bb2 +; GFX1250-GISEL-NEXT: s_endpgm +bb: + br label %bb3 + +bb2: ; preds = %bb3 + ret void + +bb3: ; preds = %bb3, %bb + %i = phi i32 [ 0, %bb ], [ %i8, %bb3 ] + %i4 = zext i32 %i to i64 + %i5 = getelementptr inbounds float, ptr %arg, i64 %i4 + %i6 = load volatile float, ptr %i5, align 4 + %i5.1 = getelementptr inbounds float, ptr %arg.1, i64 %i4 + %i6.1 = load volatile float, ptr %i5, align 4 + %i8 = add nuw nsw i32 %i, 1 + %i9 = icmp eq i32 %i8, 256 + br i1 %i9, label %bb2, label %bb3 +} + +!0 = !{i32 0, i32 1073741824} ; (1 << 30) +!1 = !{i32 0, i32 1073741825} ; (1 << 30) + 1 diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll new file mode 100644 index 0000000..32888d2 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll @@ -0,0 +1,1118 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s + +; Test using saddr addressing mode of flat_*store_* instructions. + +define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) { +; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] +; GFX1250-NEXT: s_endpgm + %voffset = load i32, ptr %voffset.ptr + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store i8 %data, ptr %gep0 + ret void +} + +; Maximum positive offset on gfx10 +define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) { +; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] offset:2047 +; GFX1250-NEXT: s_endpgm + %voffset = load i32, ptr %voffset.ptr + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 2047 + store i8 %data, ptr %gep1 + ret void +} + +; Maximum negative offset on gfx10 +define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) { +; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_load_b32 v0, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] offset:-2048 +; GFX1250-NEXT: s_endpgm + %voffset = load i32, ptr %voffset.ptr + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -2048 + store i8 %data, ptr %gep1 + ret void +} + +; -------------------------------------------------------------------------------- +; Uniformity edge cases +; -------------------------------------------------------------------------------- + +@ptr.in.lds = internal addrspace(3) global ptr undef + +; Base pointer is uniform, but also in VGPRs +define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_store_b8 v0, v1, s[0:1] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_store_b8 v[2:3], v1 +; GFX1250-GISEL-NEXT: s_endpgm + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store i8 %data, ptr %gep0 + ret void +} + +; Base pointer is uniform, but also in VGPRs, with imm offset +define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset, i8 %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2 +; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3 +; GFX1250-SDAG-NEXT: flat_store_b8 v0, v1, s[0:1] offset:-120 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 +; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2 +; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0 +; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo +; GFX1250-GISEL-NEXT: flat_store_b8 v[2:3], v1 offset:-120 +; GFX1250-GISEL-NEXT: s_endpgm + %sbase = load ptr, ptr addrspace(3) @ptr.in.lds + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -120 + store i8 %data, ptr %gep1 + ret void +} + +; -------------------------------------------------------------------------------- +; Stress various type stores +; -------------------------------------------------------------------------------- + +define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, i16 %data) { +; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store i16 %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i16 %data) { +; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store i16 %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, half %data) { +; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store half %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, half %data) { +; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store half %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store i32 %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store i32 %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, float %data) { +; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store float %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, float %data) { +; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store float %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) { +; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store ptr addrspace(3) %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) { +; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store ptr addrspace(3) %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_i64_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store i64 %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store i64 %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %voffset, double %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_f64_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store double %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, double %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store double %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i32> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2i32_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <2 x i32> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i32> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <2 x i32> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x float> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2f32_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <2 x float> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x float> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <2 x float> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x i16> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4i16_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <4 x i16> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x i16> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <4 x i16> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x half> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4f16_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <4 x half> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x half> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <4 x half> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_p1_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store ptr %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store ptr %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <3 x i32> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v3i32_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <3 x i32> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <3 x i32> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <3 x i32> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <3 x float> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v3f32_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <3 x float> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <3 x float> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <3 x float> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <6 x i16> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v6i16_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <6 x i16> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <6 x i16> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <6 x i16> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <6 x half> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v6f16_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <6 x half> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <6 x half> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2 +; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3 +; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <6 x half> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x i32> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4i32_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <4 x i32> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x i32> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <4 x i32> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x float> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4f32_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <4 x float> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x float> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <4 x float> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i64> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2i64_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <2 x i64> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i64> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <2 x i64> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x double> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2f64_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <2 x double> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x double> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <2 x double> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <8 x i16> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v8i16_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <8 x i16> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <8 x i16> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <8 x i16> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <8 x half> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v8f16_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <8 x half> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <8 x half> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <8 x half> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x ptr> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2p1_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <2 x ptr> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x ptr> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <2 x ptr> %data, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x ptr addrspace(3)> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4p3_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store <4 x ptr addrspace(3)> %data, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x ptr addrspace(3)> %data) { +; GFX1250-SDAG-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128 +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2 +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 +; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128 +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store <4 x ptr addrspace(3)> %data, ptr %gep1 + ret void +} + +; -------------------------------------------------------------------------------- +; Atomic store +; -------------------------------------------------------------------------------- + +define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt 0x0 +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store atomic i32 %data, ptr %gep0 seq_cst, align 4 + ret void +} + +define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) { +; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-NEXT: s_wait_storecnt 0x0 +; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store atomic i32 %data, ptr %gep1 seq_cst, align 4 + ret void +} + +define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i64_zext_vgpr: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + store atomic i64 %data, ptr %gep0 seq_cst, align 8 + ret void +} + +define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) { +; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128: +; GFX1250-SDAG: ; %bb.0: +; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1 +; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0 +; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-SDAG-NEXT: s_endpgm +; +; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128: +; GFX1250-GISEL: ; %bb.0: +; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2 +; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0 +; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_SYS +; GFX1250-GISEL-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + store atomic i64 %data, ptr %gep1 seq_cst, align 8 + ret void +} + +; -------------------------------------------------------------------------------- +; D16 HI store (hi 16) +; -------------------------------------------------------------------------------- + +define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) { +; GFX1250-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %data.hi = extractelement <2 x i16> %data, i32 1 + store i16 %data.hi, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) { +; GFX1250-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %data.hi = extractelement <2 x i16> %data, i32 1 + store i16 %data.hi, ptr %gep1 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) { +; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3] +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %data.hi = extractelement <2 x i16> %data, i32 1 + %data.hi.trunc = trunc i16 %data.hi to i8 + store i8 %data.hi.trunc, ptr %gep0 + ret void +} + +define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) { +; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3] offset:-128 +; GFX1250-NEXT: s_endpgm + %zext.offset = zext i32 %voffset to i64 + %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset + %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128 + %data.hi = extractelement <2 x i16> %data, i32 1 + %data.hi.trunc = trunc i16 %data.hi to i8 + store i8 %data.hi.trunc, ptr %gep1 + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll index 2f08931..8784352 100644 --- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll +++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll @@ -1872,63 +1872,48 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b, ; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v1.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0: @@ -1985,74 +1970,59 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b, ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v1.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.h ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v1 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0: @@ -2505,114 +2475,88 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> % ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1 +; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v6, v8 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.h, v1.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v4.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.h ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v4, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v4.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v4.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.h, s2 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v2.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.h, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v1, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v2.l, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v1.h, s2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0: @@ -2710,123 +2654,103 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> % ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 -; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1 +; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.h, v1.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v6, v8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v4.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v4, v5 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.h ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7 -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v4.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v4.h, s1 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.h, s2 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v2.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.h, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v2.l, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v1.h, s2 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-FAKE16-LABEL: v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0: diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll index 0742ac7..bc85dc2 100644 --- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll @@ -69,21 +69,16 @@ define bfloat @v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum(bfloat %a) #1 { ; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x4000, v0.l, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x4000, v1.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v1.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] %max = call bfloat @llvm.maximumnum.bf16(bfloat %a, bfloat 2.0) %med = call bfloat @llvm.minimumnum.bf16(bfloat %max, bfloat 4.0) @@ -196,35 +191,26 @@ define <2 x bfloat> @v_test_fmed3_r_i_i_v2bf16_minimumnum_maximumnum(<2 x bfloat ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e64 s0, v2, v2 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.h, 0x4000, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e64 s0, v1, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v2 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v2.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.h, 0x4000, v0.l, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v2 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 2.0, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v1.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v2.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v2 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 4.0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x4080, v1.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 4.0, v2 ; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, s0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] %max = call <2 x bfloat> @llvm.maximumnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> splat (bfloat 2.0)) diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll index 969c6c3..7b2d793 100644 --- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll +++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll @@ -1874,63 +1874,48 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b, ; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0: ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v1.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0: @@ -1987,74 +1972,59 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b, ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v1.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.h ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v1 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0: @@ -2510,114 +2480,88 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> % ; GFX11-SDAG-TRUE16: ; %bb.0: ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1 +; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v6, v8 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s1 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.h, v1.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v4.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.h ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v4, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v4.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2 -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v4.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.h, s2 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v2.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.h, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v3 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v1, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v2.l, s0 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v1 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v5.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v5.h, s1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6 -; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1 -; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo -; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v1.h, s2 +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo +; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-FAKE16-LABEL: v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0: @@ -2715,123 +2659,103 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> % ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 -; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1 +; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 ; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.h, v1.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v6, v8 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v4.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v4, v5 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.h ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7 -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v4.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v4.h, s1 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.h, s2 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v2.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.h, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v3 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6 -; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1 -; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v1, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v2.l, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v1 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v5.h, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v5.h, s1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l +; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0 +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v1.h, s2 +; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo ; GFX12-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-FAKE16-LABEL: v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0: diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll index ac43806..9a347d7 100644 --- a/llvm/test/CodeGen/AMDGPU/freeze.ll +++ b/llvm/test/CodeGen/AMDGPU/freeze.ll @@ -14592,5 +14592,241 @@ define void @freeze_v4i1_vcc(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb) { store <4 x i1> %freeze, ptr addrspace(1) %ptrb ret void } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX8-SDAG: {{.*}} + +define double @freeze_fabs_double(float %a, double %b, double %c) { +; GFX6-SDAG-LABEL: freeze_fabs_double: +; GFX6-SDAG: ; %bb.0: +; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-SDAG-NEXT: v_mov_b32_e32 v5, v0 +; GFX6-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2] +; GFX6-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4] +; GFX6-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX6-GISEL-LABEL: freeze_fabs_double: +; GFX6-GISEL: ; %bb.0: +; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-GISEL-NEXT: v_and_b32_e32 v5, 0x7fffffff, v0 +; GFX6-GISEL-NEXT: v_add_f64 v[0:1], v[4:5], v[1:2] +; GFX6-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], v[3:4] +; GFX6-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX7-SDAG-LABEL: freeze_fabs_double: +; GFX7-SDAG: ; %bb.0: +; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-SDAG-NEXT: v_mov_b32_e32 v5, v0 +; GFX7-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2] +; GFX7-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4] +; GFX7-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX7-GISEL-LABEL: freeze_fabs_double: +; GFX7-GISEL: ; %bb.0: +; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0x7fffffff, v0 +; GFX7-GISEL-NEXT: v_add_f64 v[0:1], v[4:5], v[1:2] +; GFX7-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], v[3:4] +; GFX7-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-SDAG-LABEL: freeze_fabs_double: +; GFX8-SDAG: ; %bb.0: +; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-SDAG-NEXT: v_mov_b32_e32 v5, v0 +; GFX8-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2] +; GFX8-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4] +; GFX8-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-GISEL-LABEL: freeze_fabs_double: +; GFX8-GISEL: ; %bb.0: +; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-GISEL-NEXT: v_and_b32_e32 v5, 0x7fffffff, v0 +; GFX8-GISEL-NEXT: v_add_f64 v[0:1], v[4:5], v[1:2] +; GFX8-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], v[3:4] +; GFX8-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-GISEL-LABEL: freeze_fabs_double: +; GFX9-GISEL: ; %bb.0: +; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-GISEL-NEXT: v_and_b32_e32 v5, 0x7fffffff, v0 +; GFX9-GISEL-NEXT: v_add_f64 v[0:1], v[4:5], v[1:2] +; GFX9-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], v[3:4] +; GFX9-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: freeze_fabs_double: +; GFX10-SDAG: ; %bb.0: +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: v_mov_b32_e32 v5, v0 +; GFX10-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2] +; GFX10-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4] +; GFX10-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-GISEL-LABEL: freeze_fabs_double: +; GFX10-GISEL: ; %bb.0: +; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-GISEL-NEXT: v_and_b32_e32 v5, 0x7fffffff, v0 +; GFX10-GISEL-NEXT: v_add_f64 v[0:1], v[4:5], v[1:2] +; GFX10-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], v[3:4] +; GFX10-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: freeze_fabs_double: +; GFX11-SDAG: ; %bb.0: +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-SDAG-NEXT: v_add_f64 v[0:1], |v[4:5]|, v[1:2] +; GFX11-SDAG-NEXT: v_add_f64 v[2:3], |v[4:5]|, v[3:4] +; GFX11-SDAG-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: freeze_fabs_double: +; GFX11-GISEL: ; %bb.0: +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0x7fffffff, v0 +; GFX11-GISEL-NEXT: v_add_f64 v[0:1], v[4:5], v[1:2] +; GFX11-GISEL-NEXT: v_add_f64 v[2:3], v[4:5], v[3:4] +; GFX11-GISEL-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3] +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] + %pv = insertelement <2 x float> poison, float %a, i32 1 + %d = bitcast <2 x float> %pv to double + %r = call double @llvm.fabs.f64(double %d) + %fr = freeze double %r + %add1 = fadd double %fr, %b + %add2 = fadd double %fr, %c + %add = fadd double %add1, %add2 + ret double %add +} + +define <4 x float> @freeze_fabs_v4float(<4 x float> %A, <4 x float> %B) { +; GFX6-SDAG-LABEL: freeze_fabs_v4float: +; GFX6-SDAG: ; %bb.0: +; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-SDAG-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX6-SDAG-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX6-SDAG-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX6-SDAG-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX6-GISEL-LABEL: freeze_fabs_v4float: +; GFX6-GISEL: ; %bb.0: +; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX6-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX6-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX6-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX6-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX6-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX6-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX7-SDAG-LABEL: freeze_fabs_v4float: +; GFX7-SDAG: ; %bb.0: +; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX7-GISEL-LABEL: freeze_fabs_v4float: +; GFX7-GISEL: ; %bb.0: +; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-SDAG-LABEL: freeze_fabs_v4float: +; GFX8-SDAG: ; %bb.0: +; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX8-SDAG-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX8-SDAG-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX8-SDAG-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-GISEL-LABEL: freeze_fabs_v4float: +; GFX8-GISEL: ; %bb.0: +; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX8-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX8-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX8-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX8-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-GISEL-LABEL: freeze_fabs_v4float: +; GFX9-GISEL: ; %bb.0: +; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: freeze_fabs_v4float: +; GFX10-SDAG: ; %bb.0: +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX10-SDAG-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-GISEL-LABEL: freeze_fabs_v4float: +; GFX10-GISEL: ; %bb.0: +; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: freeze_fabs_v4float: +; GFX11-SDAG: ; %bb.0: +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX11-SDAG-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX11-SDAG-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: freeze_fabs_v4float: +; GFX11-GISEL: ; %bb.0: +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] + %A0 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %A) + %F1 = freeze <4 x float> %A0 + %A1 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %F1) + ret <4 x float> %A1 +} diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll index a901d7f..f8ff8ef 100644 --- a/llvm/test/CodeGen/AMDGPU/function-args.ll +++ b/llvm/test/CodeGen/AMDGPU/function-args.ll @@ -1109,18 +1109,19 @@ define void @void_func_v4i8(<4 x i8> %arg0) #0 { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0 ; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h ; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -1191,19 +1192,20 @@ define void @void_func_v5i8(<5 x i8> %arg0) #0 { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 4 ; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h ; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l ; GFX11-TRUE16-NEXT: buffer_store_b8 v4, off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; @@ -1289,23 +1291,24 @@ define void @void_func_v8i8(<8 x i8> %arg0) #0 { ; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v0.h, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v0.h, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 -; GFX11-TRUE16-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4 +; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v0, v4 +; GFX11-TRUE16-NEXT: buffer_store_b64 v[1:2], off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: void_func_v8i8: @@ -1419,46 +1422,47 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 { ; GFX11-TRUE16-LABEL: void_func_v16i8: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v13.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 8, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v14.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v12.h +; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v8.h, v13.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v9.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, 0 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.l ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v13 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v13.l, v8.h, v13.l -; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v12.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.l, v9.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v5.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v0.h, v7.l +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v4.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v9, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v10.l, v8.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v11 +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v9, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v8, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v4, v5 +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v8, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v6 ; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v4, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 -; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v0, v12 +; GFX11-TRUE16-NEXT: buffer_store_b128 v[6:9], off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: void_func_v16i8: @@ -1654,85 +1658,84 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: scratch_load_d16_u8 v31, off, s32 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v12.l +; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v2.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v9.l ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.l ; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v7.h, v6.h +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v5.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v13 ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v7.l ; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v21.l -; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v28.l ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v9.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v29.l -; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v13, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v5.h, v4.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v14 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v12, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v6.l, v7.l ; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v23.l -; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v7.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v12.l, v11.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v1.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v17.l, v3.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v5.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v7.h, v6.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v8.h, v8.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v10.l, v9.h -; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v11.l, v10.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v13.l, v12.h -; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v14.l, v13.h -; GFX11-TRUE16-NEXT: v_or_b16 v11.l, v15.l, v14.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v21.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v20.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v11.h, v11.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v5 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v13, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v8.h, v8.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v0.h, v6.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v13, v14 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v4, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v15, v16 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v8 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v11, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.h, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v6 ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 16 ; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v31.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v9.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v31.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v9.l, v5.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v7, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v10.h, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v9, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v4.h, v5.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v9, v10 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v17, v18 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v12, v19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v11, v32 +; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v4.l, v8.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v9, v32 ; GFX11-TRUE16-NEXT: buffer_store_b128 v[4:7], off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0 ; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll index dd7f183..facc91a 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll @@ -4896,22 +4896,23 @@ define amdgpu_gfx void @test_call_external_void_func_v4i8_ret() #0 { ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0 ; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l ; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1 ; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 ; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33 ; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-TRUE16-NEXT: global_store_b32 v[40:41], v0, off ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33 @@ -5156,18 +5157,22 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8_ret() #0 { ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v6 ; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l ; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_lshlrev_b32 v3, 16, v1 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 4 ; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0 +; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33 +; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2 ; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v3 ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: global_store_b8 v[0:1], v4, off @@ -5175,9 +5180,6 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8_ret() #0 { ; GFX11-TRUE16-NEXT: s_clause 0x1 ; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33 ; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4 -; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0 -; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33 -; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2 ; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1 @@ -5440,36 +5442,35 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8_ret() #0 { ; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v8 ; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v5.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 ; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v3.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v4 -; GFX11-TRUE16-NEXT: global_store_b64 v[40:41], v[0:1], off -; GFX11-TRUE16-NEXT: s_clause 0x1 -; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33 -; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v1.h, v1.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4 +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v3, v5 +; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v4 ; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v0, v5 ; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0 ; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33 ; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2 +; GFX11-TRUE16-NEXT: global_store_b64 v[40:41], v[1:2], off +; GFX11-TRUE16-NEXT: s_clause 0x1 +; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33 +; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4 ; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload ; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1 @@ -5911,81 +5912,83 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8_ret() #0 { ; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1] ; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v13.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v9.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v15.l ; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v11.l -; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, 0 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l ; GFX11-TRUE16-NEXT: v_or_b16 v12.l, v1.h, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l -; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.l -; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v7.l -; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v1.h, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v3.h, v2.h -; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v5.h, v4.h -; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l -; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v6.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v14.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v9.l ; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v7, v8 -; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v28.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v12 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v31.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v30.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v0.h -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.l, v3.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.l, v3.l -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v12, v6 -; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v2.h, v1.h -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v27.l -; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v21.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff, v12 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v0.h, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v3.h, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l +; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v11.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v9, v13 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v8 +; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.h, v2.h +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v7.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v8, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v1.h, v0.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v3.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v29.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v28.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v25.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v24.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v31.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v2, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v26.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v6.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v21.l ; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v20.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v23.l -; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v17.l -; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v16.l -; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v19.l -; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v18.l -; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l -; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l -; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l -; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v3.h, v3.l -; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v9 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v10, v8 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v12, v0 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v1, v2 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v3, v7 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v11, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v6, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v0.l +; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v23.l +; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v22.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l +; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l +; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v7, v13 +; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v18.l +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v6.l, v0.h +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v19.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v6, v13 +; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v1.l, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v10 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v0, v13 ; GFX11-TRUE16-NEXT: s_clause 0x1 -; GFX11-TRUE16-NEXT: global_store_b128 v[42:43], v[7:10], off -; GFX11-TRUE16-NEXT: global_store_b128 v[40:41], v[3:6], off +; GFX11-TRUE16-NEXT: global_store_b128 v[42:43], v[6:9], off +; GFX11-TRUE16-NEXT: global_store_b128 v[40:41], v[2:5], off ; GFX11-TRUE16-NEXT: s_clause 0x3 ; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s33 ; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:4 diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll index 100a560..1f74fbd 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll @@ -12320,7 +12320,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -12334,20 +12334,22 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12467,7 +12469,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -12482,19 +12484,21 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12821,34 +12825,34 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12974,12 +12978,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -12990,19 +12993,21 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13339,34 +13344,34 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13493,12 +13498,11 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -13509,19 +13513,21 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13855,7 +13861,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -13869,18 +13875,20 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -13997,7 +14005,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -14012,17 +14020,19 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -14337,45 +14347,45 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -14487,45 +14497,46 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -14838,45 +14849,45 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -14989,45 +15000,46 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -15341,27 +15353,28 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_ ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 @@ -15461,27 +15474,28 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) @@ -15750,32 +15764,33 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -15865,33 +15880,34 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -16149,34 +16165,34 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -16304,12 +16320,11 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -16320,19 +16335,21 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -16668,46 +16685,46 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB63_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -16820,45 +16837,46 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB63_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll index faa3ee6..faa74fe 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll @@ -8741,7 +8741,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -8755,20 +8755,22 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -8888,7 +8890,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -8903,19 +8905,21 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9244,34 +9248,34 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9397,12 +9401,11 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -9413,19 +9416,21 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9764,34 +9769,34 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9918,12 +9923,11 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -9934,19 +9938,21 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10282,7 +10288,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10296,18 +10302,20 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -10424,7 +10432,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10439,17 +10447,19 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -10766,45 +10776,45 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -10916,45 +10926,46 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -11269,45 +11280,45 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11420,45 +11431,46 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -11774,27 +11786,28 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_ ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 @@ -11894,27 +11907,28 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) @@ -12185,32 +12199,33 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -12300,33 +12315,34 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12586,34 +12602,34 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12741,12 +12757,11 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -12757,19 +12772,21 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13107,46 +13124,46 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13259,45 +13276,46 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll index cb66f85..a46b012 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll @@ -8741,7 +8741,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -8755,20 +8755,22 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -8888,7 +8890,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -8903,19 +8905,21 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9244,34 +9248,34 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9397,12 +9401,11 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -9413,19 +9416,21 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9764,34 +9769,34 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9918,12 +9923,11 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -9934,19 +9938,21 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10282,7 +10288,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10296,18 +10302,20 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -10424,7 +10432,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10439,17 +10447,19 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory( ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -10766,45 +10776,45 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -10916,45 +10926,46 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -11269,45 +11280,45 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11420,45 +11431,46 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -11774,27 +11786,28 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_ ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 @@ -11894,27 +11907,28 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_ ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) @@ -12185,32 +12199,33 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_ ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -12300,33 +12315,34 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_ ; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12586,34 +12602,34 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -12741,12 +12757,11 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -12757,19 +12772,21 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13107,46 +13124,46 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13259,45 +13276,46 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX11-TRUE16-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll index f869b57..053efdc 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll @@ -9266,7 +9266,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9280,20 +9280,22 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9413,7 +9415,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -9428,19 +9430,21 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9767,34 +9771,34 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -9920,12 +9924,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -9936,19 +9939,21 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10285,34 +10290,34 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10439,12 +10444,11 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -10455,19 +10459,21 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -10801,7 +10807,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10815,18 +10821,20 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -10943,7 +10951,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 @@ -10958,17 +10966,19 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat % ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 @@ -11283,45 +11293,45 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1) ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11433,45 +11443,46 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1) ; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -11784,45 +11795,45 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1) ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -11935,45 +11946,46 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1) ; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -12287,27 +12299,28 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 ; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 @@ -12407,27 +12420,28 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) @@ -12696,32 +12710,33 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4 -; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -12811,33 +12826,34 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr ; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4 -; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5 +; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 @@ -13095,34 +13111,34 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1 ; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4 ; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13250,12 +13266,11 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1 ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 -; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 ; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -13266,19 +13281,21 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1 ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2 -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -13614,46 +13631,46 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1 ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 +; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff +; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0 -; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS +; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS -; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe ; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffe @@ -13766,45 +13783,46 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1 ; GFX11-TRUE16-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 ; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0 -; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3 +; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff -; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff +; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3 ; GFX11-TRUE16-NEXT: .p2align 6 ; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start ; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3 +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1 +; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1 -; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7 -; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7 +; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3 ; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc +; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-TRUE16-NEXT: buffer_gl1_inv ; GFX11-TRUE16-NEXT: buffer_gl0_inv -; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3 ; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll index d588f0e..723e3ef 100644 --- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll +++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll @@ -4007,6 +4007,8 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16lo_zero_hi(ptr addrspace(1 ; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] ; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_lshl_or_b32 v0, 0, 16, v0 ; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog ; ; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_i16_d16lo_zero_hi: @@ -4053,6 +4055,8 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16lo_zero_hi_immneg128(ptr a ; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] offset:-128 ; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0 ; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_lshl_or_b32 v0, 0, 16, v0 ; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog ; ; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_i16_d16lo_zero_hi_immneg128: @@ -4411,7 +4415,7 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16hi_zero_hi(ptr addrspace(1 ; GFX12-GISEL-TRUE16: ; %bb.0: ; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] ; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-GISEL-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0 ; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog ; ; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_i16_d16hi_zero_hi: @@ -4457,7 +4461,7 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16hi_zero_hi_immneg128(ptr a ; GFX12-GISEL-TRUE16: ; %bb.0: ; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] offset:-128 ; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX12-GISEL-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0 ; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog ; ; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_i16_d16hi_zero_hi_immneg128: @@ -4882,7 +4886,7 @@ define amdgpu_ps void @global_addr_64bit_lsr_iv(ptr addrspace(1) inreg %arg) { ; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0 ; GFX12-GISEL-NEXT: .LBB132_1: ; %bb3 ; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX12-GISEL-NEXT: v_add_co_u32 v4, vcc, v0, v2 ; GFX12-GISEL-NEXT: s_wait_alu 0xfffd ; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc @@ -5002,7 +5006,7 @@ define amdgpu_ps void @global_addr_64bit_lsr_iv_multiload(ptr addrspace(1) inreg ; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0 ; GFX12-GISEL-NEXT: .LBB133_1: ; %bb3 ; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX12-GISEL-NEXT: v_add_co_u32 v4, vcc, v0, v2 ; GFX12-GISEL-NEXT: s_wait_alu 0xfffd ; GFX12-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll index 5e50288..7ebd692 100644 --- a/llvm/test/CodeGen/AMDGPU/idot4u.ll +++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll @@ -2724,31 +2724,31 @@ define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1, ; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[2:3] ; GFX11-DL-TRUE16-NEXT: global_load_d16_u8 v0, v5, s[4:5] ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2) -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 24, v3 +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v3.l ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1) +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v1.l, 8, v4.l +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 24, v3 ; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v4 -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v3.l -; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v3.h, v4.h -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v4.l ; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0) ; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l -; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v6.l -; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l -; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-DL-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v2.l +; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.l +; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v3.h, v4.h +; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v2.l, v6.l +; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-DL-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-DL-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.l +; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-DL-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.h ; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-DL-TRUE16-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-DL-TRUE16-NEXT: v_or_b16 v6.h, v0.h, v1.l ; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1 -; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l -; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v6 -; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: v_or_b32_e32 v2, v2, v6 +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2 +; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l +; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v3.h, v4.h, v0.l ; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l ; GFX11-DL-TRUE16-NEXT: global_store_b8 v5, v0, s[4:5] diff --git a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir index 2cc25d8..c34c974 100644 --- a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir +++ b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir @@ -17,24 +17,22 @@ body: | liveins: $vgpr0, $sgpr4_sgpr5 ; CHECK-LABEL: name: av_mov_b32_split - ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5 + ; CHECK: liveins: $agpr3, $agpr4, $vgpr0, $sgpr4_sgpr5 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: renamable $agpr0 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec ; CHECK-NEXT: renamable $agpr1 = V_ACCVGPR_WRITE_B32_e64 1, implicit $exec ; CHECK-NEXT: renamable $agpr2 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec - ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec - ; CHECK-NEXT: renamable $agpr0 = V_ACCVGPR_WRITE_B32_e64 3, implicit $exec - ; CHECK-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec - ; CHECK-NEXT: renamable $agpr0 = V_ACCVGPR_WRITE_B32_e64 4, implicit $exec - ; CHECK-NEXT: $vgpr2 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 3, implicit $exec + ; CHECK-NEXT: $agpr3 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 4, implicit $exec + ; CHECK-NEXT: $agpr4 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0 ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr1 ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr2 - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr1, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0 - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr2, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0 + ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 $agpr3, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr0 + ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr0 %0:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec %1:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec %2:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec @@ -68,29 +66,25 @@ body: | liveins: $vgpr0, $sgpr4_sgpr5 ; CHECK-LABEL: name: v_mov_b32_split - ; CHECK: liveins: $vgpr0, $vgpr3, $vgpr4, $vgpr5, $sgpr4_sgpr5 + ; CHECK: liveins: $agpr3, $agpr4, $vgpr0, $sgpr4_sgpr5 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec ; CHECK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 1, implicit $exec ; CHECK-NEXT: renamable $vgpr2 = V_MOV_B32_e32 2, implicit $exec ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit $exec - ; CHECK-NEXT: $vgpr3 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit $exec ; CHECK-NEXT: $agpr2 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr2, implicit $exec, implicit $exec ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 3, implicit $exec - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit $exec - ; CHECK-NEXT: $vgpr4 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec + ; CHECK-NEXT: $agpr3 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 4, implicit $exec - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit $exec - ; CHECK-NEXT: $vgpr5 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr3, implicit $exec + ; CHECK-NEXT: $agpr4 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0 ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr1 ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr2 - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr4, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0 - ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr5, implicit $exec - ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0 + ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 $agpr3, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr0 + ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec + ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr0 %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec %2:vgpr_32 = V_MOV_B32_e32 2, implicit $exec diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir index c1e0d07..11de6c8 100644 --- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir +++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir @@ -27,7 +27,7 @@ # CHECK-LABEL: name: inflated_reg_class_copy_use_after_free # CHECK: S_NOP 0, implicit-def [[ORIG_REG:%[0-9]+]].sub0_sub1_sub2_sub3 # CHECK-NEXT: SI_SPILL_AV512_SAVE [[ORIG_REG]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) -# CHECK-NEXT: [[RESTORE0:%[0-9]+]]:vreg_512_align2 = SI_SPILL_V512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) +# CHECK-NEXT: [[RESTORE0:%[0-9]+]]:vreg_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) # CHECK-NEXT: early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 undef %3:vgpr_32, undef %3:vgpr_32, [[RESTORE0]], 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec # CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = COPY undef $vgpr2_vgpr3 { # CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = COPY undef $vgpr0 diff --git a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir new file mode 100644 index 0000000..0abf347 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-wmma-xdl.mir @@ -0,0 +1,67 @@ +# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -start-before=amdgpu-insert-delay-alu %s -o - | FileCheck %s + +--- +name: wmma_xdl_twoaddr_trans +tracksRegLiveness: true +body: | + bb.0: + ; CHECK-LABEL: {{^}}wmma_xdl_twoaddr_trans: + ; CHECK: %bb.0: + ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[0:7], v[0:7], v[8:15] + ; CHECK-NEXT: v_exp_f32_e32 v16, v16 + ; CHECK-NEXT: s_delay_alu instid0(TRANS32_DEP_2) + ; CHECK-NEXT: v_add_nc_u32_e32 v17, v17, v8 + liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16, $vgpr17 + $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, 0, implicit $exec + $vgpr16 = V_EXP_F32_e32 $vgpr16, implicit $exec, implicit $mode + $vgpr17 = V_ADD_U32_e32 $vgpr17, $vgpr8, implicit $exec +... + +--- +name: wmma_xdl_threeaddr_trans +tracksRegLiveness: true +body: | + bb.0: + ; CHECK-LABEL: {{^}}wmma_xdl_threeaddr_trans: + ; CHECK: %bb.0: + ; CHECK-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[8:15], v[0:7], v[0:7], v[16:23] + ; CHECK-NEXT: v_exp_f32_e32 v24, v24 + ; CHECK-NEXT: s_delay_alu instid0(TRANS32_DEP_2) + ; CHECK-NEXT: v_add_nc_u32_e32 v25, v25, v8 + liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24, $vgpr25 + $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_WMMA_F32_16X16X64_FP8_FP8_w32_threeaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, 8, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, 0, 0, 0, 0, implicit $exec + $vgpr24 = V_EXP_F32_e32 $vgpr24, implicit $exec, implicit $mode + $vgpr25 = V_ADD_U32_e32 $vgpr25, $vgpr8, implicit $exec +... + +name: swmmac_xdl_twoaddr_trans +tracksRegLiveness: true +body: | + bb.0: + ; CHECK-LABEL: {{^}}swmmac_xdl_twoaddr_trans: + ; CHECK: %bb.0: + ; CHECK-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] + ; CHECK-NEXT: v_exp_f32_e32 v30, v30 + ; CHECK-NEXT: s_delay_alu instid0(TRANS32_DEP_2) + ; CHECK-NEXT: v_add_nc_u32_e32 v31, v31, v24 + liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31 + $vgpr24_vgpr25_vgpr26_vgpr27 = V_SWMMAC_F16_16X16X128_BF8_BF8_w32_twoaddr $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23, $vgpr24_vgpr25_vgpr26_vgpr27, $vgpr28_vgpr29, 0, 0, 0, implicit $exec + $vgpr30 = V_EXP_F32_e32 $vgpr30, implicit $exec, implicit $mode + $vgpr31 = V_ADD_U32_e32 $vgpr31, $vgpr24, implicit $exec +... + +name: wmma_non_xdl_large_data_valu +tracksRegLiveness: true +body: | + bb.0: + ; CHECK-LABEL: {{^}}wmma_non_xdl_large_data_valu: + ; CHECK: %bb.0: + ; CHECK-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] matrix_b_reuse + ; CHECK-NEXT: v_exp_f32_e32 v12, v12 + ; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) + ; CHECK-NEXT: v_add_nc_u32_e32 v13, v13, v8 + liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12, $vgpr13 + $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 = V_WMMA_F32_16X16X4_F32_w32_twoaddr 8, $vgpr0_vgpr1, 8, $vgpr2_vgpr3, 8, $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11, 0, -1, 0, 0, implicit $exec + $vgpr12 = V_EXP_F32_e32 $vgpr12, implicit $exec, implicit $mode + $vgpr13 = V_ADD_U32_e32 $vgpr13, $vgpr8, implicit $exec +... diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll index 0d33400..3261e4c 100644 --- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll +++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll @@ -917,47 +917,91 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre } define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) { - ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat - ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0): - ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8 - ; DAGISEL-GFX11-WF32-NEXT: {{ $}} - ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 - ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 - ; DAGISEL-GFX11-WF32-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc - ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 - ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 - ; DAGISEL-GFX11-WF32-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec - ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF - ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] - ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) - ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0 + ; DAGISEL-GFX11-WF32-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF32-TRUE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_MOV_B16_t16_e64_]], %subreg.lo16, [[COPY]], %subreg.hi16 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[REG_SEQUENCE]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF32-TRUE16-NEXT: S_ENDPGM 0 ; - ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_cc_bfloat - ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0): - ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8 - ; DAGISEL-GFX11-WF64-NEXT: {{ $}} - ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 - ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 - ; DAGISEL-GFX11-WF64-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc - ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 - ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 - ; DAGISEL-GFX11-WF64-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec - ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF - ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] - ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) - ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0 + ; DAGISEL-GFX11-WF32-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF32-FAKE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF32-FAKE16-NEXT: S_ENDPGM 0 + ; + ; DAGISEL-GFX11-WF64-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF64-TRUE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_MOV_B16_t16_e64_]], %subreg.lo16, [[COPY]], %subreg.hi16 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[REG_SEQUENCE]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF64-TRUE16-NEXT: S_ENDPGM 0 + ; + ; DAGISEL-GFX11-WF64-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat + ; DAGISEL-GFX11-WF64-FAKE16: bb.0 (%ir-block.0): + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: liveins: $sgpr0, $vgpr8 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: {{ $}} + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304 + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]] + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`) + ; DAGISEL-GFX11-WF64-FAKE16-NEXT: S_ENDPGM 0 ; ; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat ; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0): diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll index 25d1028..4f81d35 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll @@ -8,11 +8,11 @@ ; RUN: | FileCheck -check-prefix=GCN-O3 %s -; GCN-O0: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(atomic-expand,verify,gc-lowering,lower-constant-intrinsics,unreachableblockelim,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(lower-switch,lower-invoke,unreachableblockelim,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa,require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,localstackalloc))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,phi-node-elimination,two-address-instruction,regallocfast,si-fix-vgpr-copies,remove-redundant-debug-values,fixup-statepoint-caller-saved,prolog-epilog,post-ra-pseudos,fentry-insert,xray-instrumentation,patchable-function,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),invalidate<machine-function-info>)) +; GCN-O0: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(atomic-expand,verify,gc-lowering,lower-constant-intrinsics,unreachableblockelim,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(lower-switch,lower-invoke,unreachableblockelim,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa,require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,localstackalloc))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,phi-node-elimination,two-address-instruction,regallocfast,si-fix-vgpr-copies,remove-redundant-debug-values,fixup-statepoint-caller-saved,prolog-epilog,post-ra-pseudos,si-post-ra-bundler,fentry-insert,xray-instrumentation,patchable-function,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function)) -; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),invalidate<machine-function-info>)) +; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function)) -; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),invalidate<machine-function-info>)) +; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function)) define void @empty() { ret void diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index af3241e..2a5c652 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -329,6 +329,7 @@ ; GCN-O1-NEXT: Remove dead machine instructions ; GCN-O1-NEXT: SI Shrink Instructions ; GCN-O1-NEXT: Register Usage Information Propagation +; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc ; GCN-O1-NEXT: Detect Dead Lanes ; GCN-O1-NEXT: Remove dead machine instructions ; GCN-O1-NEXT: Init Undef Pass @@ -640,6 +641,7 @@ ; GCN-O1-OPTS-NEXT: Remove dead machine instructions ; GCN-O1-OPTS-NEXT: SI Shrink Instructions ; GCN-O1-OPTS-NEXT: Register Usage Information Propagation +; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc ; GCN-O1-OPTS-NEXT: Detect Dead Lanes ; GCN-O1-OPTS-NEXT: Remove dead machine instructions ; GCN-O1-OPTS-NEXT: Init Undef Pass @@ -956,6 +958,7 @@ ; GCN-O2-NEXT: Remove dead machine instructions ; GCN-O2-NEXT: SI Shrink Instructions ; GCN-O2-NEXT: Register Usage Information Propagation +; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc ; GCN-O2-NEXT: Detect Dead Lanes ; GCN-O2-NEXT: Remove dead machine instructions ; GCN-O2-NEXT: Init Undef Pass @@ -1286,6 +1289,7 @@ ; GCN-O3-NEXT: Remove dead machine instructions ; GCN-O3-NEXT: SI Shrink Instructions ; GCN-O3-NEXT: Register Usage Information Propagation +; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc ; GCN-O3-NEXT: Detect Dead Lanes ; GCN-O3-NEXT: Remove dead machine instructions ; GCN-O3-NEXT: Init Undef Pass diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll new file mode 100644 index 0000000..091859f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll @@ -0,0 +1,33 @@ +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s +; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s + +; FIXME: GlobalISel does not work with bf16 + +declare bfloat @llvm.amdgcn.cos.bf16(bfloat) #0 + +; GCN-LABEL: {{^}}cos_bf16: +; GCN: v_cos_bf16_e32 {{v[0-9]+}}, {{s[0-9]+}} +define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 { + %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat %src) #0 + store bfloat %cos, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}cos_bf16_constant_4 +; GCN: v_cos_bf16_e32 v0, 4.0 +define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 { + %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat 4.0) #0 + store bfloat %cos, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}cos_bf16_constant_100 +; GCN: v_cos_bf16_e32 {{v[0-9]+}}, 0x42c8 +define amdgpu_kernel void @cos_bf16_constant_100(ptr addrspace(1) %out) #1 { + %cos = call bfloat @llvm.amdgcn.cos.bf16(bfloat 100.0) #0 + store bfloat %cos, ptr addrspace(1) %out, align 2 + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll new file mode 100644 index 0000000..3a55070 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll @@ -0,0 +1,305 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=SDAG-REAL16 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=SDAG-FAKE16 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s + +declare i16 @llvm.amdgcn.sat.pk4.i4.i8(i32) #0 +declare i16 @llvm.amdgcn.sat.pk4.u4.u8(i32) #0 + +define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 { +; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_v: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_clause 0x1 +; SDAG-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, s2 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_v: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_clause 0x1 +; SDAG-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v1, s2 +; SDAG-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm +; +; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_v: +; GISEL-REAL16: ; %bb.0: +; GISEL-REAL16-NEXT: s_clause 0x1 +; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0 +; GISEL-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, s2 +; GISEL-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-REAL16-NEXT: s_endpgm +; +; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_v: +; GISEL-FAKE16: ; %bb.0: +; GISEL-FAKE16-NEXT: s_clause 0x1 +; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GISEL-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v0, s2 +; GISEL-FAKE16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-FAKE16-NEXT: s_endpgm + %cvt = call i16 @llvm.amdgcn.sat.pk4.i4.i8(i32 %src) #0 + store i16 %cvt, ptr %out, align 2 + ret void +} + +define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 { +; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_s: +; SDAG-REAL16: ; %bb.1: +; SDAG-REAL16-NEXT: s_load_b32 s8, s[4:5], 0x0 +; SDAG-REAL16-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-REAL16-NEXT: s_branch .LBB1_0 +; SDAG-REAL16-NEXT: .p2align 8 +; SDAG-REAL16-NEXT: ; %bb.2: +; SDAG-REAL16-NEXT: .LBB1_0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, s8 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_s: +; SDAG-FAKE16: ; %bb.1: +; SDAG-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x0 +; SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-FAKE16-NEXT: s_branch .LBB1_0 +; SDAG-FAKE16-NEXT: .p2align 8 +; SDAG-FAKE16-NEXT: ; %bb.2: +; SDAG-FAKE16-NEXT: .LBB1_0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v1, s8 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm +; +; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_s: +; GISEL-REAL16: ; %bb.0: +; GISEL-REAL16-NEXT: s_clause 0x1 +; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0 +; GISEL-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, s2 +; GISEL-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-REAL16-NEXT: s_endpgm +; +; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_s: +; GISEL-FAKE16: ; %bb.0: +; GISEL-FAKE16-NEXT: s_clause 0x1 +; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GISEL-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v0, s2 +; GISEL-FAKE16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-FAKE16-NEXT: s_endpgm + %cvt = call i16 @llvm.amdgcn.sat.pk4.i4.i8(i32 %src) #0 + store i16 %cvt, ptr %out, align 2 + ret void +} + +define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 { +; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_i: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, 0x64 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_i: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v1, 0x64 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm +; +; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_i: +; GISEL-REAL16: ; %bb.0: +; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; GISEL-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, 0x64 +; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0 +; GISEL-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-REAL16-NEXT: s_endpgm +; +; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_i: +; GISEL-FAKE16: ; %bb.0: +; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; GISEL-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v0, 0x64 +; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GISEL-FAKE16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-FAKE16-NEXT: s_endpgm + %cvt = call i16 @llvm.amdgcn.sat.pk4.i4.i8(i32 100) #0 + store i16 %cvt, ptr %out, align 2 + ret void +} + +define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 { +; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_v: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_clause 0x1 +; SDAG-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, s2 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_v: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_clause 0x1 +; SDAG-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v1, s2 +; SDAG-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm +; +; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_v: +; GISEL-REAL16: ; %bb.0: +; GISEL-REAL16-NEXT: s_clause 0x1 +; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0 +; GISEL-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, s2 +; GISEL-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-REAL16-NEXT: s_endpgm +; +; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_v: +; GISEL-FAKE16: ; %bb.0: +; GISEL-FAKE16-NEXT: s_clause 0x1 +; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GISEL-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v0, s2 +; GISEL-FAKE16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-FAKE16-NEXT: s_endpgm + %cvt = call i16 @llvm.amdgcn.sat.pk4.u4.u8(i32 %src) #0 + store i16 %cvt, ptr %out, align 2 + ret void +} + +define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 { +; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_s: +; SDAG-REAL16: ; %bb.1: +; SDAG-REAL16-NEXT: s_load_b32 s8, s[4:5], 0x0 +; SDAG-REAL16-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-REAL16-NEXT: s_branch .LBB4_0 +; SDAG-REAL16-NEXT: .p2align 8 +; SDAG-REAL16-NEXT: ; %bb.2: +; SDAG-REAL16-NEXT: .LBB4_0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, s8 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_s: +; SDAG-FAKE16: ; %bb.1: +; SDAG-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x0 +; SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-FAKE16-NEXT: s_branch .LBB4_0 +; SDAG-FAKE16-NEXT: .p2align 8 +; SDAG-FAKE16-NEXT: ; %bb.2: +; SDAG-FAKE16-NEXT: .LBB4_0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v1, s8 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm +; +; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_s: +; GISEL-REAL16: ; %bb.0: +; GISEL-REAL16-NEXT: s_clause 0x1 +; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0 +; GISEL-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, s2 +; GISEL-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-REAL16-NEXT: s_endpgm +; +; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_s: +; GISEL-FAKE16: ; %bb.0: +; GISEL-FAKE16-NEXT: s_clause 0x1 +; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 +; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 +; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GISEL-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v0, s2 +; GISEL-FAKE16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-FAKE16-NEXT: s_endpgm + %cvt = call i16 @llvm.amdgcn.sat.pk4.u4.u8(i32 %src) #0 + store i16 %cvt, ptr %out, align 2 + ret void +} + +define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 { +; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_i: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, 0x64 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_i: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0 +; SDAG-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v1, 0x64 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm +; +; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_i: +; GISEL-REAL16: ; %bb.0: +; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; GISEL-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, 0x64 +; GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-REAL16-NEXT: s_wait_kmcnt 0x0 +; GISEL-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-REAL16-NEXT: s_endpgm +; +; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_i: +; GISEL-FAKE16: ; %bb.0: +; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; GISEL-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v0, 0x64 +; GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GISEL-FAKE16-NEXT: flat_store_b16 v1, v0, s[0:1] +; GISEL-FAKE16-NEXT: s_endpgm + %cvt = call i16 @llvm.amdgcn.sat.pk4.u4.u8(i32 100) #0 + store i16 %cvt, ptr %out, align 2 + ret void +} + +attributes #0 = { nounwind memory(none) } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll new file mode 100644 index 0000000..6304923 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll @@ -0,0 +1,33 @@ +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s +; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s + +; FIXME: GlobalISel does not work with bf16 + +declare bfloat @llvm.amdgcn.exp2.bf16(bfloat) #0 + +; GCN-LABEL: {{^}}exp_bf16: +; GCN: v_exp_bf16_e32 {{v[0-9]+}}, {{s[0-9]+}} +define amdgpu_kernel void @exp_bf16(ptr addrspace(1) %out, bfloat %src) #1 { + %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat %src) #0 + store bfloat %exp, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}exp_bf16_constant_4 +; GCN: v_exp_bf16_e32 v0, 4.0 +define amdgpu_kernel void @exp_bf16_constant_4(ptr addrspace(1) %out) #1 { + %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat 4.0) #0 + store bfloat %exp, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}exp_bf16_constant_100 +; GCN: v_exp_bf16_e32 {{v[0-9]+}}, 0x42c8 +define amdgpu_kernel void @exp_bf16_constant_100(ptr addrspace(1) %out) #1 { + %exp = call bfloat @llvm.amdgcn.exp2.bf16(bfloat 100.0) #0 + store bfloat %exp, ptr addrspace(1) %out, align 2 + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll new file mode 100644 index 0000000..a8b2077 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll @@ -0,0 +1,33 @@ +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s +; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s + +; FIXME: GlobalISel does not work with bf16 + +declare bfloat @llvm.amdgcn.log.bf16(bfloat) #0 + +; GCN-LABEL: {{^}}log_bf16: +; GCN: v_log_bf16_e32 {{v[0-9]+}}, {{s[0-9]+}} +define amdgpu_kernel void @log_bf16(ptr addrspace(1) %out, bfloat %src) #1 { + %log = call bfloat @llvm.amdgcn.log.bf16(bfloat %src) #0 + store bfloat %log, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}log_bf16_constant_4 +; GCN: v_log_bf16_e32 v0, 4.0 +define amdgpu_kernel void @log_bf16_constant_4(ptr addrspace(1) %out) #1 { + %log = call bfloat @llvm.amdgcn.log.bf16(bfloat 4.0) #0 + store bfloat %log, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}log_bf16_constant_100 +; GCN: v_log_bf16_e32 {{v[0-9]+}}, 0x42c8 +define amdgpu_kernel void @log_bf16_constant_100(ptr addrspace(1) %out) #1 { + %log = call bfloat @llvm.amdgcn.log.bf16(bfloat 100.0) #0 + store bfloat %log, ptr addrspace(1) %out, align 2 + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.form.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.form.ll new file mode 100644 index 0000000..87a7c2e --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.form.ll @@ -0,0 +1,76 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn -mcpu=gfx950 --amdgpu-mfma-vgpr-form=0 < %s | FileCheck -enable-var-scope --check-prefixes=HEURRC %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx950 --amdgpu-mfma-vgpr-form=1 < %s | FileCheck -enable-var-scope --check-prefixes=VGPRRC %s + +declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) + +define <4 x float> @default(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) { +; HEURRC-LABEL: default: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: default: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_setpc_b64 s[30:31] + %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) + ret <4 x float> %result +} + +define <4 x float> @request_agpr(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) #0 { +; HEURRC-LABEL: request_agpr: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: request_agpr: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_setpc_b64 s[30:31] + %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) + ret <4 x float> %result +} + +define <4 x float> @request_no_agpr(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) #1 { +; HEURRC-LABEL: request_no_agpr: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: request_no_agpr: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_setpc_b64 s[30:31] + %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) + ret <4 x float> %result +} + +attributes #0 = { "amdgpu-agpr-alloc"="32,256" } +attributes #1 = { "amdgpu-agpr-alloc"="0,0" } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll index 4628a9c..866dba77 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SDAG %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 --amdgpu-mfma-vgpr-form=0 < %s | FileCheck -enable-var-scope --check-prefixes=HEURRC %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 --amdgpu-mfma-vgpr-form=1 < %s | FileCheck -enable-var-scope --check-prefixes=VGPRRC %s declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half>, <8 x half>, <16 x float>, i32 immarg, i32 immarg, i32 immarg) @@ -25,6 +27,48 @@ define <4 x float> @test_mfma_f32_16x16x32_f16(<8 x half> %arg0, <8 x half> %arg ; GCN-NEXT: v_accvgpr_read_b32 v2, a2 ; GCN-NEXT: v_accvgpr_read_b32 v3, a3 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_f16: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_f32_16x16x32_f16: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_f32_16x16x32_f16: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) ret <4 x float> %result } @@ -45,6 +89,48 @@ define <4 x float> @test_mfma_f32_16x16x32_f16__flags(<8 x half> %arg0, <8 x hal ; GCN-NEXT: v_accvgpr_read_b32 v2, a2 ; GCN-NEXT: v_accvgpr_read_b32 v3, a3 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_f16__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_f32_16x16x32_f16__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_f32_16x16x32_f16__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1) ret <4 x float> %result } @@ -91,6 +177,84 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrsp ; GISEL-NEXT: s_nop 6 ; GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s0 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s1 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s2 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s3 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPR-NEXT: v_mov_b32_e32 v12, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPR-NEXT: s_endpgm %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) store <4 x float> %result, ptr addrspace(1) %out ret void @@ -138,6 +302,84 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr ; GISEL-NEXT: s_nop 6 ; GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s0 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s1 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s2 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s3 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPR-NEXT: v_mov_b32_e32 v12, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPR-NEXT: s_endpgm %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1) store <4 x float> %result, ptr addrspace(1) %out ret void @@ -271,6 +513,258 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal ; GISEL-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[16:17], 16 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: v_mov_b64_e32 v[18:19], 0 +; HEURRC-NEXT: v_mov_b32_e32 v8, s16 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: v_mov_b32_e32 v9, s17 +; HEURRC-NEXT: v_mov_b32_e32 v10, s18 +; HEURRC-NEXT: v_mov_b32_e32 v11, s19 +; HEURRC-NEXT: s_nop 4 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], 16 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], 0 +; VGPRRC-NEXT: v_mov_b32_e32 v40, s16 +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] +; VGPRRC-NEXT: v_mov_b32_e32 v41, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s19 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 +; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 +; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_32x32x16_f16: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b64_e32 v[12:13], 48 +; AGPR-NEXT: v_mov_b64_e32 v[14:15], 32 +; AGPR-NEXT: v_mov_b64_e32 v[16:17], 16 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: v_mov_b64_e32 v[18:19], 0 +; AGPR-NEXT: v_mov_b32_e32 v8, s16 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: v_mov_b32_e32 v9, s17 +; AGPR-NEXT: v_mov_b32_e32 v10, s18 +; AGPR-NEXT: v_mov_b32_e32 v11, s19 +; AGPR-NEXT: s_nop 4 +; AGPR-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_32x32x16_f16: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b64_e32 v[44:45], 48 +; VGPR-NEXT: v_mov_b64_e32 v[46:47], 32 +; VGPR-NEXT: v_mov_b64_e32 v[48:49], 16 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[50:51], 0 +; VGPR-NEXT: v_mov_b32_e32 v40, s16 +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] +; VGPR-NEXT: v_mov_b32_e32 v41, s17 +; VGPR-NEXT: v_mov_b32_e32 v42, s18 +; VGPR-NEXT: v_mov_b32_e32 v43, s19 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v0, s20 +; VGPR-NEXT: v_mov_b32_e32 v1, s21 +; VGPR-NEXT: v_mov_b32_e32 v2, s22 +; VGPR-NEXT: v_mov_b32_e32 v3, s23 +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s8 +; VGPR-NEXT: v_mov_b32_e32 v1, s9 +; VGPR-NEXT: v_mov_b32_e32 v2, s10 +; VGPR-NEXT: v_mov_b32_e32 v3, s11 +; VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s12 +; VGPR-NEXT: v_mov_b32_e32 v1, s13 +; VGPR-NEXT: v_mov_b32_e32 v2, s14 +; VGPR-NEXT: v_mov_b32_e32 v3, s15 +; VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0) store volatile <16 x float> %result, ptr addrspace(1) null store volatile <16 x float> %arg2, ptr addrspace(1) null @@ -401,6 +895,258 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, < ; GISEL-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[16:17], 16 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: v_mov_b64_e32 v[18:19], 0 +; HEURRC-NEXT: v_mov_b32_e32 v8, s16 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: v_mov_b32_e32 v9, s17 +; HEURRC-NEXT: v_mov_b32_e32 v10, s18 +; HEURRC-NEXT: v_mov_b32_e32 v11, s19 +; HEURRC-NEXT: s_nop 4 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[48:49], 16 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[50:51], 0 +; VGPRRC-NEXT: v_mov_b32_e32 v40, s16 +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s19 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 +; VGPRRC-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 +; VGPRRC-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b64_e32 v[12:13], 48 +; AGPR-NEXT: v_mov_b64_e32 v[14:15], 32 +; AGPR-NEXT: v_mov_b64_e32 v[16:17], 16 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: v_mov_b64_e32 v[18:19], 0 +; AGPR-NEXT: v_mov_b32_e32 v8, s16 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: v_mov_b32_e32 v9, s17 +; AGPR-NEXT: v_mov_b32_e32 v10, s18 +; AGPR-NEXT: v_mov_b32_e32 v11, s19 +; AGPR-NEXT: s_nop 4 +; AGPR-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b64_e32 v[44:45], 48 +; VGPR-NEXT: v_mov_b64_e32 v[46:47], 32 +; VGPR-NEXT: v_mov_b64_e32 v[48:49], 16 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[50:51], 0 +; VGPR-NEXT: v_mov_b32_e32 v40, s16 +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1 +; VGPR-NEXT: v_mov_b32_e32 v41, s17 +; VGPR-NEXT: v_mov_b32_e32 v42, s18 +; VGPR-NEXT: v_mov_b32_e32 v43, s19 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v0, s20 +; VGPR-NEXT: v_mov_b32_e32 v1, s21 +; VGPR-NEXT: v_mov_b32_e32 v2, s22 +; VGPR-NEXT: v_mov_b32_e32 v3, s23 +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s8 +; VGPR-NEXT: v_mov_b32_e32 v1, s9 +; VGPR-NEXT: v_mov_b32_e32 v2, s10 +; VGPR-NEXT: v_mov_b32_e32 v3, s11 +; VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s12 +; VGPR-NEXT: v_mov_b32_e32 v1, s13 +; VGPR-NEXT: v_mov_b32_e32 v2, s14 +; VGPR-NEXT: v_mov_b32_e32 v3, s15 +; VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 2, i32 3, i32 1) store volatile <16 x float> %result, ptr addrspace(1) null store volatile <16 x float> %arg2, ptr addrspace(1) null @@ -448,6 +1194,134 @@ define <16 x float> @test_mfma_f32_32x32x16_f16__mac(<8 x half> %arg0, <8 x half ; GCN-NEXT: v_accvgpr_read_b32 v14, a14 ; GCN-NEXT: v_accvgpr_read_b32 v15, a15 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__mac: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 3 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4 +; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5 +; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6 +; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7 +; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8 +; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9 +; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10 +; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11 +; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12 +; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13 +; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14 +; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__mac: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 3 +; VGPRRC-NEXT: v_mov_b32_e32 v0, v8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, v9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, v10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, v11 +; VGPRRC-NEXT: v_mov_b32_e32 v4, v12 +; VGPRRC-NEXT: v_mov_b32_e32 v5, v13 +; VGPRRC-NEXT: v_mov_b32_e32 v6, v14 +; VGPRRC-NEXT: v_mov_b32_e32 v7, v15 +; VGPRRC-NEXT: v_mov_b32_e32 v8, v16 +; VGPRRC-NEXT: v_mov_b32_e32 v9, v17 +; VGPRRC-NEXT: v_mov_b32_e32 v10, v18 +; VGPRRC-NEXT: v_mov_b32_e32 v11, v19 +; VGPRRC-NEXT: v_mov_b32_e32 v12, v20 +; VGPRRC-NEXT: v_mov_b32_e32 v13, v21 +; VGPRRC-NEXT: v_mov_b32_e32 v14, v22 +; VGPRRC-NEXT: v_mov_b32_e32 v15, v23 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__mac: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, v12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, v13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, v14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, v15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, v16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, v17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, v18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, v19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, v20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, v21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, v22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, v23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 3 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: v_accvgpr_read_b32 v4, a4 +; AGPR-NEXT: v_accvgpr_read_b32 v5, a5 +; AGPR-NEXT: v_accvgpr_read_b32 v6, a6 +; AGPR-NEXT: v_accvgpr_read_b32 v7, a7 +; AGPR-NEXT: v_accvgpr_read_b32 v8, a8 +; AGPR-NEXT: v_accvgpr_read_b32 v9, a9 +; AGPR-NEXT: v_accvgpr_read_b32 v10, a10 +; AGPR-NEXT: v_accvgpr_read_b32 v11, a11 +; AGPR-NEXT: v_accvgpr_read_b32 v12, a12 +; AGPR-NEXT: v_accvgpr_read_b32 v13, a13 +; AGPR-NEXT: v_accvgpr_read_b32 v14, a14 +; AGPR-NEXT: v_accvgpr_read_b32 v15, a15 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__mac: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 3 +; VGPR-NEXT: v_mov_b32_e32 v0, v8 +; VGPR-NEXT: v_mov_b32_e32 v1, v9 +; VGPR-NEXT: v_mov_b32_e32 v2, v10 +; VGPR-NEXT: v_mov_b32_e32 v3, v11 +; VGPR-NEXT: v_mov_b32_e32 v4, v12 +; VGPR-NEXT: v_mov_b32_e32 v5, v13 +; VGPR-NEXT: v_mov_b32_e32 v6, v14 +; VGPR-NEXT: v_mov_b32_e32 v7, v15 +; VGPR-NEXT: v_mov_b32_e32 v8, v16 +; VGPR-NEXT: v_mov_b32_e32 v9, v17 +; VGPR-NEXT: v_mov_b32_e32 v10, v18 +; VGPR-NEXT: v_mov_b32_e32 v11, v19 +; VGPR-NEXT: v_mov_b32_e32 v12, v20 +; VGPR-NEXT: v_mov_b32_e32 v13, v21 +; VGPR-NEXT: v_mov_b32_e32 v14, v22 +; VGPR-NEXT: v_mov_b32_e32 v15, v23 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0) ret <16 x float> %result } @@ -493,6 +1367,134 @@ define <16 x float> @test_mfma_f32_32x32x16_f16__mac__flags(<8 x half> %arg0, <8 ; GCN-NEXT: v_accvgpr_read_b32 v14, a14 ; GCN-NEXT: v_accvgpr_read_b32 v15, a15 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__mac__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 3 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4 +; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5 +; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6 +; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7 +; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8 +; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9 +; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10 +; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11 +; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12 +; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13 +; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14 +; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__mac__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 3 +; VGPRRC-NEXT: v_mov_b32_e32 v0, v8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, v9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, v10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, v11 +; VGPRRC-NEXT: v_mov_b32_e32 v4, v12 +; VGPRRC-NEXT: v_mov_b32_e32 v5, v13 +; VGPRRC-NEXT: v_mov_b32_e32 v6, v14 +; VGPRRC-NEXT: v_mov_b32_e32 v7, v15 +; VGPRRC-NEXT: v_mov_b32_e32 v8, v16 +; VGPRRC-NEXT: v_mov_b32_e32 v9, v17 +; VGPRRC-NEXT: v_mov_b32_e32 v10, v18 +; VGPRRC-NEXT: v_mov_b32_e32 v11, v19 +; VGPRRC-NEXT: v_mov_b32_e32 v12, v20 +; VGPRRC-NEXT: v_mov_b32_e32 v13, v21 +; VGPRRC-NEXT: v_mov_b32_e32 v14, v22 +; VGPRRC-NEXT: v_mov_b32_e32 v15, v23 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__mac__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, v12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, v13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, v14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, v15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, v16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, v17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, v18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, v19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, v20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, v21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, v22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, v23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 3 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: v_accvgpr_read_b32 v4, a4 +; AGPR-NEXT: v_accvgpr_read_b32 v5, a5 +; AGPR-NEXT: v_accvgpr_read_b32 v6, a6 +; AGPR-NEXT: v_accvgpr_read_b32 v7, a7 +; AGPR-NEXT: v_accvgpr_read_b32 v8, a8 +; AGPR-NEXT: v_accvgpr_read_b32 v9, a9 +; AGPR-NEXT: v_accvgpr_read_b32 v10, a10 +; AGPR-NEXT: v_accvgpr_read_b32 v11, a11 +; AGPR-NEXT: v_accvgpr_read_b32 v12, a12 +; AGPR-NEXT: v_accvgpr_read_b32 v13, a13 +; AGPR-NEXT: v_accvgpr_read_b32 v14, a14 +; AGPR-NEXT: v_accvgpr_read_b32 v15, a15 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__mac__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 3 +; VGPR-NEXT: v_mov_b32_e32 v0, v8 +; VGPR-NEXT: v_mov_b32_e32 v1, v9 +; VGPR-NEXT: v_mov_b32_e32 v2, v10 +; VGPR-NEXT: v_mov_b32_e32 v3, v11 +; VGPR-NEXT: v_mov_b32_e32 v4, v12 +; VGPR-NEXT: v_mov_b32_e32 v5, v13 +; VGPR-NEXT: v_mov_b32_e32 v6, v14 +; VGPR-NEXT: v_mov_b32_e32 v7, v15 +; VGPR-NEXT: v_mov_b32_e32 v8, v16 +; VGPR-NEXT: v_mov_b32_e32 v9, v17 +; VGPR-NEXT: v_mov_b32_e32 v10, v18 +; VGPR-NEXT: v_mov_b32_e32 v11, v19 +; VGPR-NEXT: v_mov_b32_e32 v12, v20 +; VGPR-NEXT: v_mov_b32_e32 v13, v21 +; VGPR-NEXT: v_mov_b32_e32 v14, v22 +; VGPR-NEXT: v_mov_b32_e32 v15, v23 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 1, i32 1, i32 1) ret <16 x float> %result } @@ -615,6 +1617,246 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, ; GISEL-NEXT: global_store_dwordx4 v24, a[28:31], s[0:1] offset:48 sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: v_mov_b32_e32 v12, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 +; HEURRC-NEXT: v_mov_b32_e32 v8, s20 +; HEURRC-NEXT: v_mov_b32_e32 v9, s21 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] +; HEURRC-NEXT: v_mov_b32_e32 v10, s22 +; HEURRC-NEXT: v_mov_b32_e32 v11, s23 +; HEURRC-NEXT: v_mov_b32_e32 v0, s16 +; HEURRC-NEXT: v_mov_b32_e32 v1, s17 +; HEURRC-NEXT: v_mov_b32_e32 v2, s18 +; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: v_mov_b32_e32 v44, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPRRC-NEXT: v_mov_b32_e32 v40, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s21 +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] +; VGPRRC-NEXT: v_mov_b32_e32 v42, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s23 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 2 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: v_mov_b32_e32 v12, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; AGPR-NEXT: v_accvgpr_write_b32 a31, s23 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; AGPR-NEXT: v_accvgpr_write_b32 a30, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a29, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a28, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a27, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a26, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a25, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a24, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a23, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a22, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a21, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a20, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a19, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a18, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a17, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a16, s8 +; AGPR-NEXT: v_mov_b32_e32 v8, s20 +; AGPR-NEXT: v_mov_b32_e32 v9, s21 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] +; AGPR-NEXT: v_mov_b32_e32 v10, s22 +; AGPR-NEXT: v_mov_b32_e32 v11, s23 +; AGPR-NEXT: v_mov_b32_e32 v0, s16 +; AGPR-NEXT: v_mov_b32_e32 v1, s17 +; AGPR-NEXT: v_mov_b32_e32 v2, s18 +; AGPR-NEXT: v_mov_b32_e32 v3, s19 +; AGPR-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: v_mov_b32_e32 v44, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPR-NEXT: v_mov_b32_e32 v40, s20 +; VGPR-NEXT: v_mov_b32_e32 v41, s21 +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] +; VGPR-NEXT: v_mov_b32_e32 v42, s22 +; VGPR-NEXT: v_mov_b32_e32 v43, s23 +; VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 2 +; VGPR-NEXT: v_mov_b32_e32 v16, s16 +; VGPR-NEXT: v_mov_b32_e32 v17, s17 +; VGPR-NEXT: v_mov_b32_e32 v18, s18 +; VGPR-NEXT: v_mov_b32_e32 v19, s19 +; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s12 +; VGPR-NEXT: v_mov_b32_e32 v17, s13 +; VGPR-NEXT: v_mov_b32_e32 v18, s14 +; VGPR-NEXT: v_mov_b32_e32 v19, s15 +; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s8 +; VGPR-NEXT: v_mov_b32_e32 v17, s9 +; VGPR-NEXT: v_mov_b32_e32 v18, s10 +; VGPR-NEXT: v_mov_b32_e32 v19, s11 +; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0) store volatile <16 x float> %arg2, ptr addrspace(1) %out store volatile <16 x float> %result, ptr addrspace(1) %out @@ -739,6 +1981,246 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> ; GISEL-NEXT: global_store_dwordx4 v24, a[28:31], s[0:1] offset:48 sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: v_mov_b32_e32 v12, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 +; HEURRC-NEXT: v_mov_b32_e32 v8, s20 +; HEURRC-NEXT: v_mov_b32_e32 v9, s21 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 +; HEURRC-NEXT: v_mov_b32_e32 v10, s22 +; HEURRC-NEXT: v_mov_b32_e32 v11, s23 +; HEURRC-NEXT: v_mov_b32_e32 v0, s16 +; HEURRC-NEXT: v_mov_b32_e32 v1, s17 +; HEURRC-NEXT: v_mov_b32_e32 v2, s18 +; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: v_mov_b32_e32 v44, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPRRC-NEXT: v_mov_b32_e32 v40, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v41, s21 +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3 +; VGPRRC-NEXT: v_mov_b32_e32 v42, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v43, s23 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 2 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 +; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: v_mov_b32_e32 v12, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; AGPR-NEXT: v_accvgpr_write_b32 a31, s23 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; AGPR-NEXT: v_accvgpr_write_b32 a30, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a29, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a28, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a27, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a26, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a25, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a24, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a23, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a22, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a21, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a20, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a19, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a18, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a17, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a16, s8 +; AGPR-NEXT: v_mov_b32_e32 v8, s20 +; AGPR-NEXT: v_mov_b32_e32 v9, s21 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 +; AGPR-NEXT: v_mov_b32_e32 v10, s22 +; AGPR-NEXT: v_mov_b32_e32 v11, s23 +; AGPR-NEXT: v_mov_b32_e32 v0, s16 +; AGPR-NEXT: v_mov_b32_e32 v1, s17 +; AGPR-NEXT: v_mov_b32_e32 v2, s18 +; AGPR-NEXT: v_mov_b32_e32 v3, s19 +; AGPR-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: v_mov_b32_e32 v44, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27] +; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25] +; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31] +; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29] +; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPR-NEXT: v_mov_b32_e32 v40, s20 +; VGPR-NEXT: v_mov_b32_e32 v41, s21 +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3 +; VGPR-NEXT: v_mov_b32_e32 v42, s22 +; VGPR-NEXT: v_mov_b32_e32 v43, s23 +; VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 2 +; VGPR-NEXT: v_mov_b32_e32 v16, s16 +; VGPR-NEXT: v_mov_b32_e32 v17, s17 +; VGPR-NEXT: v_mov_b32_e32 v18, s18 +; VGPR-NEXT: v_mov_b32_e32 v19, s19 +; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s12 +; VGPR-NEXT: v_mov_b32_e32 v17, s13 +; VGPR-NEXT: v_mov_b32_e32 v18, s14 +; VGPR-NEXT: v_mov_b32_e32 v19, s15 +; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s8 +; VGPR-NEXT: v_mov_b32_e32 v17, s9 +; VGPR-NEXT: v_mov_b32_e32 v18, s10 +; VGPR-NEXT: v_mov_b32_e32 v19, s11 +; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 1, i32 2, i32 3) store volatile <16 x float> %arg2, ptr addrspace(1) %out store volatile <16 x float> %result, ptr addrspace(1) %out @@ -819,6 +2301,136 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac(<8 x half> %ar ; GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 2 +; HEURRC-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; HEURRC-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; HEURRC-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25] +; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27] +; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29] +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] +; VGPRRC-NEXT: v_mov_b32_e32 v16, 0 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 2 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] +; AGPR-NEXT: v_mov_b32_e32 v0, 0 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 2 +; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[24:25] +; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[26:27] +; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[28:29] +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[30:31] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] +; VGPR-NEXT: v_mov_b32_e32 v16, 0 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 2 +; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPR-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0) store <16 x float> %result, ptr addrspace(1) %out ret void @@ -898,6 +2510,136 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac_flags(<8 x hal ; GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1 +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 2 +; HEURRC-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; HEURRC-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; HEURRC-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25] +; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27] +; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29] +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1 +; VGPRRC-NEXT: v_mov_b32_e32 v16, 0 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 2 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1 +; AGPR-NEXT: v_mov_b32_e32 v0, 0 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 2 +; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[24:25] +; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[26:27] +; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[28:29] +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[30:31] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1 +; VGPR-NEXT: v_mov_b32_e32 v16, 0 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 2 +; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPR-NEXT: s_endpgm %result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 3, i32 2, i32 1) store <16 x float> %result, ptr addrspace(1) %out ret void @@ -925,6 +2667,48 @@ define <4 x i32> @test_mfma_i32_16x16x64_i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 ; GCN-NEXT: v_accvgpr_read_b32 v2, a2 ; GCN-NEXT: v_accvgpr_read_b32 v3, a3 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_i32_16x16x64_i8: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_i32_16x16x64_i8: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_i32_16x16x64_i8: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0) ret <4 x i32> %result } @@ -945,6 +2729,48 @@ define <4 x i32> @test_mfma_i32_16x16x64_i8__flags(<4 x i32> %arg0, <4 x i32> %a ; GCN-NEXT: v_accvgpr_read_b32 v2, a2 ; GCN-NEXT: v_accvgpr_read_b32 v3, a3 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_i32_16x16x64_i8__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_i32_16x16x64_i8__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_i32_16x16x64_i8__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 1, i32 1, i32 1) ret <4 x i32> %result } @@ -995,6 +2821,104 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspa ; GISEL-NEXT: s_nop 6 ; GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: v_mov_b32_e32 v4, s12 +; HEURRC-NEXT: v_mov_b32_e32 v5, s13 +; HEURRC-NEXT: v_mov_b32_e32 v6, s14 +; HEURRC-NEXT: v_mov_b32_e32 v7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 +; VGPRRC-NEXT: v_mov_b32_e32 v4, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v5, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v6, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v7, s15 +; VGPRRC-NEXT: v_mov_b32_e32 v8, s0 +; VGPRRC-NEXT: v_mov_b32_e32 v9, s1 +; VGPRRC-NEXT: v_mov_b32_e32 v10, s2 +; VGPRRC-NEXT: v_mov_b32_e32 v11, s3 +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: v_mov_b32_e32 v4, s12 +; AGPR-NEXT: v_mov_b32_e32 v5, s13 +; AGPR-NEXT: v_mov_b32_e32 v6, s14 +; AGPR-NEXT: v_mov_b32_e32 v7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a0, s0 +; AGPR-NEXT: v_accvgpr_write_b32 a1, s1 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s2 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s3 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPR-NEXT: v_mov_b32_e32 v12, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v0, s8 +; VGPR-NEXT: v_mov_b32_e32 v1, s9 +; VGPR-NEXT: v_mov_b32_e32 v2, s10 +; VGPR-NEXT: v_mov_b32_e32 v3, s11 +; VGPR-NEXT: v_mov_b32_e32 v4, s12 +; VGPR-NEXT: v_mov_b32_e32 v5, s13 +; VGPR-NEXT: v_mov_b32_e32 v6, s14 +; VGPR-NEXT: v_mov_b32_e32 v7, s15 +; VGPR-NEXT: v_mov_b32_e32 v8, s0 +; VGPR-NEXT: v_mov_b32_e32 v9, s1 +; VGPR-NEXT: v_mov_b32_e32 v10, s2 +; VGPR-NEXT: v_mov_b32_e32 v11, s3 +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPR-NEXT: s_endpgm %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0) store <4 x i32> %result, ptr addrspace(1) %out ret void @@ -1046,6 +2970,104 @@ define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr ; GISEL-NEXT: s_nop 6 ; GISEL-NEXT: global_store_dwordx4 v0, a[0:3], s[6:7] ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: v_mov_b32_e32 v4, s12 +; HEURRC-NEXT: v_mov_b32_e32 v5, s13 +; HEURRC-NEXT: v_mov_b32_e32 v6, s14 +; HEURRC-NEXT: v_mov_b32_e32 v7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 +; VGPRRC-NEXT: v_mov_b32_e32 v4, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v5, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v6, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v7, s15 +; VGPRRC-NEXT: v_mov_b32_e32 v8, s0 +; VGPRRC-NEXT: v_mov_b32_e32 v9, s1 +; VGPRRC-NEXT: v_mov_b32_e32 v10, s2 +; VGPRRC-NEXT: v_mov_b32_e32 v11, s3 +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: v_mov_b32_e32 v4, s12 +; AGPR-NEXT: v_mov_b32_e32 v5, s13 +; AGPR-NEXT: v_mov_b32_e32 v6, s14 +; AGPR-NEXT: v_mov_b32_e32 v7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a0, s0 +; AGPR-NEXT: v_accvgpr_write_b32 a1, s1 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s2 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s3 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPR-NEXT: v_mov_b32_e32 v12, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v0, s8 +; VGPR-NEXT: v_mov_b32_e32 v1, s9 +; VGPR-NEXT: v_mov_b32_e32 v2, s10 +; VGPR-NEXT: v_mov_b32_e32 v3, s11 +; VGPR-NEXT: v_mov_b32_e32 v4, s12 +; VGPR-NEXT: v_mov_b32_e32 v5, s13 +; VGPR-NEXT: v_mov_b32_e32 v6, s14 +; VGPR-NEXT: v_mov_b32_e32 v7, s15 +; VGPR-NEXT: v_mov_b32_e32 v8, s0 +; VGPR-NEXT: v_mov_b32_e32 v9, s1 +; VGPR-NEXT: v_mov_b32_e32 v10, s2 +; VGPR-NEXT: v_mov_b32_e32 v11, s3 +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPR-NEXT: s_endpgm %result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 3, i32 2, i32 1) store <4 x i32> %result, ptr addrspace(1) %out ret void @@ -1187,6 +3209,282 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> ; GISEL-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s24 +; HEURRC-NEXT: v_mov_b32_e32 v1, s25 +; HEURRC-NEXT: v_mov_b32_e32 v2, s26 +; HEURRC-NEXT: v_mov_b32_e32 v3, s27 +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v4, s28 +; HEURRC-NEXT: v_mov_b32_e32 v5, s29 +; HEURRC-NEXT: v_mov_b32_e32 v6, s30 +; HEURRC-NEXT: v_mov_b32_e32 v7, s31 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0 +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] +; HEURRC-NEXT: v_mov_b32_e32 v0, s16 +; HEURRC-NEXT: v_mov_b32_e32 v1, s17 +; HEURRC-NEXT: v_mov_b32_e32 v2, s18 +; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v32, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v33, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v34, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v35, s27 +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b32_e32 v36, s28 +; VGPRRC-NEXT: v_mov_b32_e32 v37, s29 +; VGPRRC-NEXT: v_mov_b32_e32 v38, s30 +; VGPRRC-NEXT: v_mov_b32_e32 v39, s31 +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 3 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v0, s16 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s19 +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_32x32x32_i8: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b64_e32 v[8:9], 48 +; AGPR-NEXT: v_mov_b64_e32 v[10:11], 32 +; AGPR-NEXT: v_mov_b64_e32 v[12:13], 16 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s24 +; AGPR-NEXT: v_mov_b32_e32 v1, s25 +; AGPR-NEXT: v_mov_b32_e32 v2, s26 +; AGPR-NEXT: v_mov_b32_e32 v3, s27 +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_mov_b32_e32 v4, s28 +; AGPR-NEXT: v_mov_b32_e32 v5, s29 +; AGPR-NEXT: v_mov_b32_e32 v6, s30 +; AGPR-NEXT: v_mov_b32_e32 v7, s31 +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: v_mov_b64_e32 v[14:15], 0 +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] +; AGPR-NEXT: v_mov_b32_e32 v0, s16 +; AGPR-NEXT: v_mov_b32_e32 v1, s17 +; AGPR-NEXT: v_mov_b32_e32 v2, s18 +; AGPR-NEXT: v_mov_b32_e32 v3, s19 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_32x32x32_i8: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b64_e32 v[40:41], 48 +; VGPR-NEXT: v_mov_b64_e32 v[42:43], 32 +; VGPR-NEXT: v_mov_b64_e32 v[44:45], 16 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v32, s24 +; VGPR-NEXT: v_mov_b32_e32 v33, s25 +; VGPR-NEXT: v_mov_b32_e32 v34, s26 +; VGPR-NEXT: v_mov_b32_e32 v35, s27 +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b32_e32 v36, s28 +; VGPR-NEXT: v_mov_b32_e32 v37, s29 +; VGPR-NEXT: v_mov_b32_e32 v38, s30 +; VGPR-NEXT: v_mov_b32_e32 v39, s31 +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 3 +; VGPR-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v0, s16 +; VGPR-NEXT: v_mov_b32_e32 v1, s17 +; VGPR-NEXT: v_mov_b32_e32 v2, s18 +; VGPR-NEXT: v_mov_b32_e32 v3, s19 +; VGPR-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s20 +; VGPR-NEXT: v_mov_b32_e32 v1, s21 +; VGPR-NEXT: v_mov_b32_e32 v2, s22 +; VGPR-NEXT: v_mov_b32_e32 v3, s23 +; VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s8 +; VGPR-NEXT: v_mov_b32_e32 v1, s9 +; VGPR-NEXT: v_mov_b32_e32 v2, s10 +; VGPR-NEXT: v_mov_b32_e32 v3, s11 +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s12 +; VGPR-NEXT: v_mov_b32_e32 v1, s13 +; VGPR-NEXT: v_mov_b32_e32 v2, s14 +; VGPR-NEXT: v_mov_b32_e32 v3, s15 +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0) store volatile <16 x i32> %result, ptr addrspace(1) null store volatile <16 x i32> %arg2, ptr addrspace(1) null @@ -1323,6 +3621,282 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 ; GISEL-NEXT: global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48 +; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32 +; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s24 +; HEURRC-NEXT: v_mov_b32_e32 v1, s25 +; HEURRC-NEXT: v_mov_b32_e32 v2, s26 +; HEURRC-NEXT: v_mov_b32_e32 v3, s27 +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v4, s28 +; HEURRC-NEXT: v_mov_b32_e32 v5, s29 +; HEURRC-NEXT: v_mov_b32_e32 v6, s30 +; HEURRC-NEXT: v_mov_b32_e32 v7, s31 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0 +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 +; HEURRC-NEXT: v_mov_b32_e32 v0, s16 +; HEURRC-NEXT: v_mov_b32_e32 v1, s17 +; HEURRC-NEXT: v_mov_b32_e32 v2, s18 +; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48 +; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32 +; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v32, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v33, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v34, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v35, s27 +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b32_e32 v36, s28 +; VGPRRC-NEXT: v_mov_b32_e32 v37, s29 +; VGPRRC-NEXT: v_mov_b32_e32 v38, s30 +; VGPRRC-NEXT: v_mov_b32_e32 v39, s31 +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 3 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v0, s16 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s19 +; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s23 +; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s11 +; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v0, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v1, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v2, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v3, s15 +; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b64_e32 v[8:9], 48 +; AGPR-NEXT: v_mov_b64_e32 v[10:11], 32 +; AGPR-NEXT: v_mov_b64_e32 v[12:13], 16 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s24 +; AGPR-NEXT: v_mov_b32_e32 v1, s25 +; AGPR-NEXT: v_mov_b32_e32 v2, s26 +; AGPR-NEXT: v_mov_b32_e32 v3, s27 +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_mov_b32_e32 v4, s28 +; AGPR-NEXT: v_mov_b32_e32 v5, s29 +; AGPR-NEXT: v_mov_b32_e32 v6, s30 +; AGPR-NEXT: v_mov_b32_e32 v7, s31 +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: v_mov_b64_e32 v[14:15], 0 +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1 +; AGPR-NEXT: v_mov_b32_e32 v0, s16 +; AGPR-NEXT: v_mov_b32_e32 v1, s17 +; AGPR-NEXT: v_mov_b32_e32 v2, s18 +; AGPR-NEXT: v_mov_b32_e32 v3, s19 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b64_e32 v[40:41], 48 +; VGPR-NEXT: v_mov_b64_e32 v[42:43], 32 +; VGPR-NEXT: v_mov_b64_e32 v[44:45], 16 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v32, s24 +; VGPR-NEXT: v_mov_b32_e32 v33, s25 +; VGPR-NEXT: v_mov_b32_e32 v34, s26 +; VGPR-NEXT: v_mov_b32_e32 v35, s27 +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b32_e32 v36, s28 +; VGPR-NEXT: v_mov_b32_e32 v37, s29 +; VGPR-NEXT: v_mov_b32_e32 v38, s30 +; VGPR-NEXT: v_mov_b32_e32 v39, s31 +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[46:47], 0 +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 3 +; VGPR-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v0, s16 +; VGPR-NEXT: v_mov_b32_e32 v1, s17 +; VGPR-NEXT: v_mov_b32_e32 v2, s18 +; VGPR-NEXT: v_mov_b32_e32 v3, s19 +; VGPR-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s20 +; VGPR-NEXT: v_mov_b32_e32 v1, s21 +; VGPR-NEXT: v_mov_b32_e32 v2, s22 +; VGPR-NEXT: v_mov_b32_e32 v3, s23 +; VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s8 +; VGPR-NEXT: v_mov_b32_e32 v1, s9 +; VGPR-NEXT: v_mov_b32_e32 v2, s10 +; VGPR-NEXT: v_mov_b32_e32 v3, s11 +; VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v0, s12 +; VGPR-NEXT: v_mov_b32_e32 v1, s13 +; VGPR-NEXT: v_mov_b32_e32 v2, s14 +; VGPR-NEXT: v_mov_b32_e32 v3, s15 +; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 2, i32 3, i32 1) store volatile <16 x i32> %result, ptr addrspace(1) null store volatile <16 x i32> %arg2, ptr addrspace(1) null @@ -1370,6 +3944,134 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac(<4 x i32> %arg0, <4 x i32> %ar ; GCN-NEXT: v_accvgpr_read_b32 v14, a14 ; GCN-NEXT: v_accvgpr_read_b32 v15, a15 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__mac: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 3 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4 +; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5 +; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6 +; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7 +; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8 +; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9 +; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10 +; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11 +; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12 +; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13 +; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14 +; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__mac: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 3 +; VGPRRC-NEXT: v_mov_b32_e32 v0, v8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, v9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, v10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, v11 +; VGPRRC-NEXT: v_mov_b32_e32 v4, v12 +; VGPRRC-NEXT: v_mov_b32_e32 v5, v13 +; VGPRRC-NEXT: v_mov_b32_e32 v6, v14 +; VGPRRC-NEXT: v_mov_b32_e32 v7, v15 +; VGPRRC-NEXT: v_mov_b32_e32 v8, v16 +; VGPRRC-NEXT: v_mov_b32_e32 v9, v17 +; VGPRRC-NEXT: v_mov_b32_e32 v10, v18 +; VGPRRC-NEXT: v_mov_b32_e32 v11, v19 +; VGPRRC-NEXT: v_mov_b32_e32 v12, v20 +; VGPRRC-NEXT: v_mov_b32_e32 v13, v21 +; VGPRRC-NEXT: v_mov_b32_e32 v14, v22 +; VGPRRC-NEXT: v_mov_b32_e32 v15, v23 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__mac: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, v12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, v13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, v14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, v15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, v16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, v17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, v18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, v19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, v20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, v21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, v22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, v23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 3 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: v_accvgpr_read_b32 v4, a4 +; AGPR-NEXT: v_accvgpr_read_b32 v5, a5 +; AGPR-NEXT: v_accvgpr_read_b32 v6, a6 +; AGPR-NEXT: v_accvgpr_read_b32 v7, a7 +; AGPR-NEXT: v_accvgpr_read_b32 v8, a8 +; AGPR-NEXT: v_accvgpr_read_b32 v9, a9 +; AGPR-NEXT: v_accvgpr_read_b32 v10, a10 +; AGPR-NEXT: v_accvgpr_read_b32 v11, a11 +; AGPR-NEXT: v_accvgpr_read_b32 v12, a12 +; AGPR-NEXT: v_accvgpr_read_b32 v13, a13 +; AGPR-NEXT: v_accvgpr_read_b32 v14, a14 +; AGPR-NEXT: v_accvgpr_read_b32 v15, a15 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__mac: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 3 +; VGPR-NEXT: v_mov_b32_e32 v0, v8 +; VGPR-NEXT: v_mov_b32_e32 v1, v9 +; VGPR-NEXT: v_mov_b32_e32 v2, v10 +; VGPR-NEXT: v_mov_b32_e32 v3, v11 +; VGPR-NEXT: v_mov_b32_e32 v4, v12 +; VGPR-NEXT: v_mov_b32_e32 v5, v13 +; VGPR-NEXT: v_mov_b32_e32 v6, v14 +; VGPR-NEXT: v_mov_b32_e32 v7, v15 +; VGPR-NEXT: v_mov_b32_e32 v8, v16 +; VGPR-NEXT: v_mov_b32_e32 v9, v17 +; VGPR-NEXT: v_mov_b32_e32 v10, v18 +; VGPR-NEXT: v_mov_b32_e32 v11, v19 +; VGPR-NEXT: v_mov_b32_e32 v12, v20 +; VGPR-NEXT: v_mov_b32_e32 v13, v21 +; VGPR-NEXT: v_mov_b32_e32 v14, v22 +; VGPR-NEXT: v_mov_b32_e32 v15, v23 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0) ret <16 x i32> %result } @@ -1415,6 +4117,134 @@ define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i ; GCN-NEXT: v_accvgpr_read_b32 v14, a14 ; GCN-NEXT: v_accvgpr_read_b32 v15, a15 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__mac__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 3 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4 +; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5 +; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6 +; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7 +; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8 +; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9 +; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10 +; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11 +; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12 +; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13 +; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14 +; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__mac__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 3 +; VGPRRC-NEXT: v_mov_b32_e32 v0, v8 +; VGPRRC-NEXT: v_mov_b32_e32 v1, v9 +; VGPRRC-NEXT: v_mov_b32_e32 v2, v10 +; VGPRRC-NEXT: v_mov_b32_e32 v3, v11 +; VGPRRC-NEXT: v_mov_b32_e32 v4, v12 +; VGPRRC-NEXT: v_mov_b32_e32 v5, v13 +; VGPRRC-NEXT: v_mov_b32_e32 v6, v14 +; VGPRRC-NEXT: v_mov_b32_e32 v7, v15 +; VGPRRC-NEXT: v_mov_b32_e32 v8, v16 +; VGPRRC-NEXT: v_mov_b32_e32 v9, v17 +; VGPRRC-NEXT: v_mov_b32_e32 v10, v18 +; VGPRRC-NEXT: v_mov_b32_e32 v11, v19 +; VGPRRC-NEXT: v_mov_b32_e32 v12, v20 +; VGPRRC-NEXT: v_mov_b32_e32 v13, v21 +; VGPRRC-NEXT: v_mov_b32_e32 v14, v22 +; VGPRRC-NEXT: v_mov_b32_e32 v15, v23 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__mac__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, v12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, v13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, v14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, v15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, v16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, v17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, v18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, v19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, v20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, v21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, v22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, v23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 3 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: v_accvgpr_read_b32 v4, a4 +; AGPR-NEXT: v_accvgpr_read_b32 v5, a5 +; AGPR-NEXT: v_accvgpr_read_b32 v6, a6 +; AGPR-NEXT: v_accvgpr_read_b32 v7, a7 +; AGPR-NEXT: v_accvgpr_read_b32 v8, a8 +; AGPR-NEXT: v_accvgpr_read_b32 v9, a9 +; AGPR-NEXT: v_accvgpr_read_b32 v10, a10 +; AGPR-NEXT: v_accvgpr_read_b32 v11, a11 +; AGPR-NEXT: v_accvgpr_read_b32 v12, a12 +; AGPR-NEXT: v_accvgpr_read_b32 v13, a13 +; AGPR-NEXT: v_accvgpr_read_b32 v14, a14 +; AGPR-NEXT: v_accvgpr_read_b32 v15, a15 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__mac__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 3 +; VGPR-NEXT: v_mov_b32_e32 v0, v8 +; VGPR-NEXT: v_mov_b32_e32 v1, v9 +; VGPR-NEXT: v_mov_b32_e32 v2, v10 +; VGPR-NEXT: v_mov_b32_e32 v3, v11 +; VGPR-NEXT: v_mov_b32_e32 v4, v12 +; VGPR-NEXT: v_mov_b32_e32 v5, v13 +; VGPR-NEXT: v_mov_b32_e32 v6, v14 +; VGPR-NEXT: v_mov_b32_e32 v7, v15 +; VGPR-NEXT: v_mov_b32_e32 v8, v16 +; VGPR-NEXT: v_mov_b32_e32 v9, v17 +; VGPR-NEXT: v_mov_b32_e32 v10, v18 +; VGPR-NEXT: v_mov_b32_e32 v11, v19 +; VGPR-NEXT: v_mov_b32_e32 v12, v20 +; VGPR-NEXT: v_mov_b32_e32 v13, v21 +; VGPR-NEXT: v_mov_b32_e32 v14, v22 +; VGPR-NEXT: v_mov_b32_e32 v15, v23 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 1, i32 1, i32 1) ret <16 x i32> %result } @@ -1544,6 +4374,274 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 ; GISEL-NEXT: global_store_dwordx4 v24, a[28:31], s[0:1] offset:48 sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b32_e32 v4, s24 +; HEURRC-NEXT: v_mov_b32_e32 v5, s25 +; HEURRC-NEXT: v_mov_b32_e32 v6, s26 +; HEURRC-NEXT: v_mov_b32_e32 v7, s27 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23 +; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s16 +; HEURRC-NEXT: v_mov_b32_e32 v1, s17 +; HEURRC-NEXT: v_mov_b32_e32 v2, s18 +; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: v_mov_b32_e32 v40, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v32, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v33, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v34, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v35, s23 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b32_e32 v36, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v37, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v38, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v39, s27 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] +; VGPRRC-NEXT: s_nop 6 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s23 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b32_e32 v4, s24 +; AGPR-NEXT: v_mov_b32_e32 v5, s25 +; AGPR-NEXT: v_mov_b32_e32 v6, s26 +; AGPR-NEXT: v_mov_b32_e32 v7, s27 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a31, s23 +; AGPR-NEXT: v_accvgpr_write_b32 a30, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a29, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a28, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a27, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a26, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a25, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a24, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a23, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a22, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a21, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a20, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a19, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a18, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a17, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a16, s8 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s16 +; AGPR-NEXT: v_mov_b32_e32 v1, s17 +; AGPR-NEXT: v_mov_b32_e32 v2, s18 +; AGPR-NEXT: v_mov_b32_e32 v3, s19 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: v_mov_b32_e32 v40, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v32, s20 +; VGPR-NEXT: v_mov_b32_e32 v33, s21 +; VGPR-NEXT: v_mov_b32_e32 v34, s22 +; VGPR-NEXT: v_mov_b32_e32 v35, s23 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b32_e32 v36, s24 +; VGPR-NEXT: v_mov_b32_e32 v37, s25 +; VGPR-NEXT: v_mov_b32_e32 v38, s26 +; VGPR-NEXT: v_mov_b32_e32 v39, s27 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] +; VGPR-NEXT: s_nop 6 +; VGPR-NEXT: v_mov_b32_e32 v16, s20 +; VGPR-NEXT: v_mov_b32_e32 v17, s21 +; VGPR-NEXT: v_mov_b32_e32 v18, s22 +; VGPR-NEXT: v_mov_b32_e32 v19, s23 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s16 +; VGPR-NEXT: v_mov_b32_e32 v17, s17 +; VGPR-NEXT: v_mov_b32_e32 v18, s18 +; VGPR-NEXT: v_mov_b32_e32 v19, s19 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s12 +; VGPR-NEXT: v_mov_b32_e32 v17, s13 +; VGPR-NEXT: v_mov_b32_e32 v18, s14 +; VGPR-NEXT: v_mov_b32_e32 v19, s15 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s8 +; VGPR-NEXT: v_mov_b32_e32 v17, s9 +; VGPR-NEXT: v_mov_b32_e32 v18, s10 +; VGPR-NEXT: v_mov_b32_e32 v19, s11 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0) store volatile <16 x i32> %arg2, ptr addrspace(1) %out store volatile <16 x i32> %result, ptr addrspace(1) %out @@ -1675,6 +4773,274 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %a ; GISEL-NEXT: global_store_dwordx4 v24, a[28:31], s[0:1] offset:48 sc0 sc1 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b32_e32 v4, s24 +; HEURRC-NEXT: v_mov_b32_e32 v5, s25 +; HEURRC-NEXT: v_mov_b32_e32 v6, s26 +; HEURRC-NEXT: v_mov_b32_e32 v7, s27 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23 +; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s16 +; HEURRC-NEXT: v_mov_b32_e32 v1, s17 +; HEURRC-NEXT: v_mov_b32_e32 v2, s18 +; HEURRC-NEXT: v_mov_b32_e32 v3, s19 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s12 +; HEURRC-NEXT: v_mov_b32_e32 v1, s13 +; HEURRC-NEXT: v_mov_b32_e32 v2, s14 +; HEURRC-NEXT: v_mov_b32_e32 v3, s15 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_nop 0 +; HEURRC-NEXT: v_mov_b32_e32 v0, s8 +; HEURRC-NEXT: v_mov_b32_e32 v1, s9 +; HEURRC-NEXT: v_mov_b32_e32 v2, s10 +; HEURRC-NEXT: v_mov_b32_e32 v3, s11 +; HEURRC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; HEURRC-NEXT: s_waitcnt vmcnt(0) +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: v_mov_b32_e32 v40, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v32, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v33, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v34, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v35, s23 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b32_e32 v36, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v37, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v38, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v39, s27 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3 +; VGPRRC-NEXT: s_nop 6 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s23 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s16 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s17 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s18 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s19 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s12 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s13 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s14 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s15 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_nop 0 +; VGPRRC-NEXT: v_mov_b32_e32 v16, s8 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s9 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s10 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s11 +; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPRRC-NEXT: s_waitcnt vmcnt(0) +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b32_e32 v4, s24 +; AGPR-NEXT: v_mov_b32_e32 v5, s25 +; AGPR-NEXT: v_mov_b32_e32 v6, s26 +; AGPR-NEXT: v_mov_b32_e32 v7, s27 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a31, s23 +; AGPR-NEXT: v_accvgpr_write_b32 a30, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a29, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a28, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a27, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a26, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a25, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a24, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a23, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a22, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a21, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a20, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a19, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a18, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a17, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a16, s8 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3 +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s16 +; AGPR-NEXT: v_mov_b32_e32 v1, s17 +; AGPR-NEXT: v_mov_b32_e32 v2, s18 +; AGPR-NEXT: v_mov_b32_e32 v3, s19 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s12 +; AGPR-NEXT: v_mov_b32_e32 v1, s13 +; AGPR-NEXT: v_mov_b32_e32 v2, s14 +; AGPR-NEXT: v_mov_b32_e32 v3, s15 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_nop 0 +; AGPR-NEXT: v_mov_b32_e32 v0, s8 +; AGPR-NEXT: v_mov_b32_e32 v1, s9 +; AGPR-NEXT: v_mov_b32_e32 v2, s10 +; AGPR-NEXT: v_mov_b32_e32 v3, s11 +; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1 +; AGPR-NEXT: s_waitcnt vmcnt(0) +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: v_mov_b32_e32 v40, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v32, s20 +; VGPR-NEXT: v_mov_b32_e32 v33, s21 +; VGPR-NEXT: v_mov_b32_e32 v34, s22 +; VGPR-NEXT: v_mov_b32_e32 v35, s23 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b32_e32 v36, s24 +; VGPR-NEXT: v_mov_b32_e32 v37, s25 +; VGPR-NEXT: v_mov_b32_e32 v38, s26 +; VGPR-NEXT: v_mov_b32_e32 v39, s27 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23] +; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3 +; VGPR-NEXT: s_nop 6 +; VGPR-NEXT: v_mov_b32_e32 v16, s20 +; VGPR-NEXT: v_mov_b32_e32 v17, s21 +; VGPR-NEXT: v_mov_b32_e32 v18, s22 +; VGPR-NEXT: v_mov_b32_e32 v19, s23 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s16 +; VGPR-NEXT: v_mov_b32_e32 v17, s17 +; VGPR-NEXT: v_mov_b32_e32 v18, s18 +; VGPR-NEXT: v_mov_b32_e32 v19, s19 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s12 +; VGPR-NEXT: v_mov_b32_e32 v17, s13 +; VGPR-NEXT: v_mov_b32_e32 v18, s14 +; VGPR-NEXT: v_mov_b32_e32 v19, s15 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_nop 0 +; VGPR-NEXT: v_mov_b32_e32 v16, s8 +; VGPR-NEXT: v_mov_b32_e32 v17, s9 +; VGPR-NEXT: v_mov_b32_e32 v18, s10 +; VGPR-NEXT: v_mov_b32_e32 v19, s11 +; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1 +; VGPR-NEXT: s_waitcnt vmcnt(0) +; VGPR-NEXT: s_endpgm %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 1, i32 2, i32 3) store volatile <16 x i32> %arg2, ptr addrspace(1) %out store volatile <16 x i32> %result, ptr addrspace(1) %out @@ -1760,6 +5126,156 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0 ; GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b32_e32 v4, s24 +; HEURRC-NEXT: v_mov_b32_e32 v5, s25 +; HEURRC-NEXT: v_mov_b32_e32 v6, s26 +; HEURRC-NEXT: v_mov_b32_e32 v7, s27 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 2 +; HEURRC-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; HEURRC-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; HEURRC-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v16, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s23 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b32_e32 v20, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v21, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v22, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v23, s27 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] +; VGPRRC-NEXT: v_mov_b32_e32 v16, 0 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 2 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b32_e32 v4, s24 +; AGPR-NEXT: v_mov_b32_e32 v5, s25 +; AGPR-NEXT: v_mov_b32_e32 v6, s26 +; AGPR-NEXT: v_mov_b32_e32 v7, s27 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] +; AGPR-NEXT: v_mov_b32_e32 v0, 0 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 2 +; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v16, s20 +; VGPR-NEXT: v_mov_b32_e32 v17, s21 +; VGPR-NEXT: v_mov_b32_e32 v18, s22 +; VGPR-NEXT: v_mov_b32_e32 v19, s23 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b32_e32 v20, s24 +; VGPR-NEXT: v_mov_b32_e32 v21, s25 +; VGPR-NEXT: v_mov_b32_e32 v22, s26 +; VGPR-NEXT: v_mov_b32_e32 v23, s27 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] +; VGPR-NEXT: v_mov_b32_e32 v16, 0 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 2 +; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPR-NEXT: s_endpgm %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0) store <16 x i32> %result, ptr addrspace(1) %out ret void @@ -1844,6 +5360,156 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> ; GISEL-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 ; GISEL-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 ; GISEL-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b32_e32 v0, s20 +; HEURRC-NEXT: v_mov_b32_e32 v1, s21 +; HEURRC-NEXT: v_mov_b32_e32 v2, s22 +; HEURRC-NEXT: v_mov_b32_e32 v3, s23 +; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; HEURRC-NEXT: v_mov_b32_e32 v4, s24 +; HEURRC-NEXT: v_mov_b32_e32 v5, s25 +; HEURRC-NEXT: v_mov_b32_e32 v6, s26 +; HEURRC-NEXT: v_mov_b32_e32 v7, s27 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11 +; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12 +; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13 +; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14 +; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15 +; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16 +; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17 +; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18 +; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19 +; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20 +; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21 +; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22 +; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1 +; HEURRC-NEXT: v_mov_b32_e32 v0, 0 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: s_nop 2 +; HEURRC-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; HEURRC-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; HEURRC-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; HEURRC-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b32_e32 v16, s20 +; VGPRRC-NEXT: v_mov_b32_e32 v17, s21 +; VGPRRC-NEXT: v_mov_b32_e32 v18, s22 +; VGPRRC-NEXT: v_mov_b32_e32 v19, s23 +; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPRRC-NEXT: v_mov_b32_e32 v20, s24 +; VGPRRC-NEXT: v_mov_b32_e32 v21, s25 +; VGPRRC-NEXT: v_mov_b32_e32 v22, s26 +; VGPRRC-NEXT: v_mov_b32_e32 v23, s27 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1 +; VGPRRC-NEXT: v_mov_b32_e32 v16, 0 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: s_nop 2 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b32_e32 v0, s20 +; AGPR-NEXT: v_mov_b32_e32 v1, s21 +; AGPR-NEXT: v_mov_b32_e32 v2, s22 +; AGPR-NEXT: v_mov_b32_e32 v3, s23 +; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; AGPR-NEXT: v_mov_b32_e32 v4, s24 +; AGPR-NEXT: v_mov_b32_e32 v5, s25 +; AGPR-NEXT: v_mov_b32_e32 v6, s26 +; AGPR-NEXT: v_mov_b32_e32 v7, s27 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, s8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, s9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s11 +; AGPR-NEXT: v_accvgpr_write_b32 a4, s12 +; AGPR-NEXT: v_accvgpr_write_b32 a5, s13 +; AGPR-NEXT: v_accvgpr_write_b32 a6, s14 +; AGPR-NEXT: v_accvgpr_write_b32 a7, s15 +; AGPR-NEXT: v_accvgpr_write_b32 a8, s16 +; AGPR-NEXT: v_accvgpr_write_b32 a9, s17 +; AGPR-NEXT: v_accvgpr_write_b32 a10, s18 +; AGPR-NEXT: v_accvgpr_write_b32 a11, s19 +; AGPR-NEXT: v_accvgpr_write_b32 a12, s20 +; AGPR-NEXT: v_accvgpr_write_b32 a13, s21 +; AGPR-NEXT: v_accvgpr_write_b32 a14, s22 +; AGPR-NEXT: v_accvgpr_write_b32 a15, s23 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1 +; AGPR-NEXT: v_mov_b32_e32 v0, 0 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: s_nop 2 +; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48 +; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32 +; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16 +; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24 +; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b32_e32 v16, s20 +; VGPR-NEXT: v_mov_b32_e32 v17, s21 +; VGPR-NEXT: v_mov_b32_e32 v18, s22 +; VGPR-NEXT: v_mov_b32_e32 v19, s23 +; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64 +; VGPR-NEXT: v_mov_b32_e32 v20, s24 +; VGPR-NEXT: v_mov_b32_e32 v21, s25 +; VGPR-NEXT: v_mov_b32_e32 v22, s26 +; VGPR-NEXT: v_mov_b32_e32 v23, s27 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19] +; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21] +; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1 +; VGPR-NEXT: v_mov_b32_e32 v16, 0 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: s_nop 2 +; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; VGPR-NEXT: s_endpgm %result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 3, i32 2, i32 1) store <16 x i32> %result, ptr addrspace(1) %out ret void @@ -1871,6 +5537,48 @@ define <4 x float> @test_mfma_f32_16x16x32_bf16(<8 x bfloat> %arg0, <8 x bfloat> ; GCN-NEXT: v_accvgpr_read_b32 v2, a2 ; GCN-NEXT: v_accvgpr_read_b32 v3, a3 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_f32_16x16x32_bf16: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_f32_16x16x32_bf16: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) ret <4 x float> %result } @@ -1891,6 +5599,48 @@ define <4 x float> @test_mfma_f32_16x16x32_bf16__flags(<8 x bfloat> %arg0, <8 x ; GCN-NEXT: v_accvgpr_read_b32 v2, a2 ; GCN-NEXT: v_accvgpr_read_b32 v3, a3 ; GCN-NEXT: s_setpc_b64 s[30:31] +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8 +; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0 +; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1 +; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2 +; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3 +; HEURRC-NEXT: s_setpc_b64 s[30:31] +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1 +; VGPRRC-NEXT: s_setpc_b64 s[30:31] +; AGPR-LABEL: test_mfma_f32_16x16x32_bf16__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; AGPR-NEXT: v_accvgpr_write_b32 a0, v8 +; AGPR-NEXT: v_accvgpr_write_b32 a1, v9 +; AGPR-NEXT: v_accvgpr_write_b32 a2, v10 +; AGPR-NEXT: v_accvgpr_write_b32 a3, v11 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: v_accvgpr_read_b32 v0, a0 +; AGPR-NEXT: v_accvgpr_read_b32 v1, a1 +; AGPR-NEXT: v_accvgpr_read_b32 v2, a2 +; AGPR-NEXT: v_accvgpr_read_b32 v3, a3 +; AGPR-NEXT: s_setpc_b64 s[30:31] +; VGPR-LABEL: test_mfma_f32_16x16x32_bf16__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1 +; VGPR-NEXT: s_setpc_b64 s[30:31] %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1) ret <4 x float> %result } @@ -1916,6 +5666,84 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd(ptr addrs ; GCN-NEXT: s_nop 7 ; GCN-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] ; GCN-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s0 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s1 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s2 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s3 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPR-NEXT: v_mov_b32_e32 v12, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPR-NEXT: s_endpgm %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0) store <4 x float> %result, ptr addrspace(1) %out ret void @@ -1942,6 +5770,84 @@ define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags(pt ; GCN-NEXT: s_nop 7 ; GCN-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] ; GCN-NEXT: s_endpgm +; +; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags: +; HEURRC: ; %bb.0: +; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; HEURRC-NEXT: v_mov_b32_e32 v8, 0 +; HEURRC-NEXT: s_waitcnt lgkmcnt(0) +; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; HEURRC-NEXT: v_accvgpr_write_b32 a0, s0 +; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; HEURRC-NEXT: v_accvgpr_write_b32 a1, s1 +; HEURRC-NEXT: v_accvgpr_write_b32 a2, s2 +; HEURRC-NEXT: v_accvgpr_write_b32 a3, s3 +; HEURRC-NEXT: s_nop 1 +; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; HEURRC-NEXT: s_nop 7 +; HEURRC-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; HEURRC-NEXT: s_endpgm +; +; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags: +; VGPRRC: ; %bb.0: +; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPRRC-NEXT: v_mov_b32_e32 v12, 0 +; VGPRRC-NEXT: s_waitcnt lgkmcnt(0) +; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPRRC-NEXT: s_nop 1 +; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPRRC-NEXT: s_nop 7 +; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPRRC-NEXT: s_endpgm +; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags: +; AGPR: ; %bb.0: +; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; AGPR-NEXT: v_mov_b32_e32 v8, 0 +; AGPR-NEXT: s_waitcnt lgkmcnt(0) +; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; AGPR-NEXT: v_accvgpr_write_b32 a0, s0 +; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; AGPR-NEXT: v_accvgpr_write_b32 a1, s1 +; AGPR-NEXT: v_accvgpr_write_b32 a2, s2 +; AGPR-NEXT: v_accvgpr_write_b32 a3, s3 +; AGPR-NEXT: s_nop 1 +; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1 +; AGPR-NEXT: s_nop 7 +; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7] +; AGPR-NEXT: s_endpgm +; VGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags: +; VGPR: ; %bb.0: +; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34 +; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54 +; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; VGPR-NEXT: v_mov_b32_e32 v12, 0 +; VGPR-NEXT: s_waitcnt lgkmcnt(0) +; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9] +; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11] +; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13] +; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3] +; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15] +; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1] +; VGPR-NEXT: s_nop 1 +; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1 +; VGPR-NEXT: s_nop 7 +; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7] +; VGPR-NEXT: s_endpgm %result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1) store <4 x float> %result, ptr addrspace(1) %out ret void diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll index 67ae05e..561eaca 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll @@ -4365,8 +4365,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm: ; NOLIT-SRCC: ; %bb.0: ; %bb ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 0 @@ -4465,8 +4465,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll index 8140866..ed6a02b 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GCN %s -; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s ; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s @@ -17,6 +19,18 @@ define { i32, i32 } @v_permlane16_swap_b32_vv(i32 %vdst_old, i32 %src0_old) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vv: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vv: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 false, i1 false) ret { i32, i32 } %v } @@ -29,6 +43,22 @@ define { i32, i32 } @v_permlane16_swap_b32_vi(i32 %vdst_old) { ; GCN-NEXT: s_nop 1 ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vi: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_mov_b32_e32 v1, 1 +; GFX950-NEXT: s_nop 1 +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vi: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_mov_b32_e32 v1, 1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 1, i1 false, i1 false) ret { i32, i32 } %v } @@ -41,6 +71,22 @@ define { i32, i32 } @v_permlane16_swap_b32_vl(i32 %vdst_old) { ; GCN-NEXT: s_nop 1 ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vl: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_mov_b32_e32 v1, 0xc1d1 +; GFX950-NEXT: s_nop 1 +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vl: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_mov_b32_e32 v1, 0xc1d1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 49617, i1 false, i1 false) ret { i32, i32 } %v } @@ -54,6 +100,23 @@ define { i32, i32 } @v_permlane16_swap_b32_iv(i32 %src0_old) { ; GCN-NEXT: s_nop 1 ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_iv: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_mov_b32_e32 v1, v0 +; GFX950-NEXT: v_mov_b32_e32 v0, 1 +; GFX950-NEXT: s_nop 1 +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_iv: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 1, i32 %src0_old, i1 false, i1 false) ret { i32, i32 } %v } @@ -67,6 +130,23 @@ define { i32, i32 } @v_permlane16_swap_b32_ss(i32 inreg %vdst_old, i32 inreg %sr ; GCN-NEXT: s_nop 1 ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_ss: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_mov_b32_e32 v0, s0 +; GFX950-NEXT: v_mov_b32_e32 v1, s1 +; GFX950-NEXT: s_nop 1 +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_ss: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 false, i1 false) ret { i32, i32 } %v } @@ -80,6 +160,23 @@ define { i32, i32 } @v_permlane16_swap_b32_sv(i32 inreg %vdst_old, i32 %src0_old ; GCN-NEXT: s_nop 1 ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_sv: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_mov_b32_e32 v1, v0 +; GFX950-NEXT: v_mov_b32_e32 v0, s0 +; GFX950-NEXT: s_nop 1 +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_sv: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 false, i1 false) ret { i32, i32 } %v } @@ -92,6 +189,22 @@ define { i32, i32 } @v_permlane16_swap_b32_vs(i32 %vdst_old, i32 inreg %src0_old ; GCN-NEXT: s_nop 1 ; GCN-NEXT: v_permlane16_swap_b32_e32 v0, v1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vs: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_mov_b32_e32 v1, s0 +; GFX950-NEXT: s_nop 1 +; GFX950-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vs: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_mov_b32_e32 v1, s0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_permlane16_swap_b32_e32 v0, v1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 false, i1 false) ret { i32, i32 } %v } @@ -102,6 +215,18 @@ define { i32, i32 } @v_permlane16_swap_b32_vv_fi(i32 %vdst_old, i32 %src0_old) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_permlane16_swap_b32_e64 v0, v1 fi:1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vv_fi: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_permlane16_swap_b32_e64 v0, v1 fi:1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vv_fi: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_permlane16_swap_b32_e64 v0, v1 fi:1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 true, i1 false) ret { i32, i32 } %v } @@ -112,6 +237,18 @@ define { i32, i32 } @v_permlane16_swap_b32_vv_bc(i32 %vdst_old, i32 %src0_old) { ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_permlane16_swap_b32_e64 v0, v1 bound_ctrl:1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vv_bc: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_permlane16_swap_b32_e64 v0, v1 bound_ctrl:1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vv_bc: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_permlane16_swap_b32_e64 v0, v1 bound_ctrl:1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 false, i1 true) ret { i32, i32 } %v } @@ -122,6 +259,18 @@ define { i32, i32 } @v_permlane16_swap_b32_vv_fi_bc(i32 %vdst_old, i32 %src0_old ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GCN-NEXT: v_permlane16_swap_b32_e64 v0, v1 bound_ctrl:1 fi:1 ; GCN-NEXT: s_setpc_b64 s[30:31] +; GFX950-LABEL: v_permlane16_swap_b32_vv_fi_bc: +; GFX950: ; %bb.0: +; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX950-NEXT: v_permlane16_swap_b32_e64 v0, v1 bound_ctrl:1 fi:1 +; GFX950-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-LABEL: v_permlane16_swap_b32_vv_fi_bc: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: v_permlane16_swap_b32_e64 v0, v1 bound_ctrl:1 fi:1 +; GFX1250-NEXT: s_set_pc_i64 s[30:31] %v = call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %vdst_old, i32 %src0_old, i1 true, i1 true) ret { i32, i32 } %v } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll index 2faf375..465414c 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll @@ -1,5 +1,7 @@ ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s declare i32 @llvm.amdgcn.prng.b32(i32) #0 @@ -29,4 +31,4 @@ define amdgpu_kernel void @prng_b32_constant_100(ptr addrspace(1) %out) #1 { attributes #0 = { nounwind readnone } -attributes #1 = { nounwind }
\ No newline at end of file +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll index 3c49d0b..199494d 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll @@ -1,10 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; xUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-TRUE16 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=SDAG-TRUE16 %s ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefix=SDAG-FAKE16 %s ; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefix=GI-TRUE16 %s ; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefix=GI-FAKE16 %s -; FIXME: t16 doesn't work at the moment because the store of s16 under t16 mode fails to select. ; FIXME: GlobalISel does not work with bf16 declare bfloat @llvm.amdgcn.rcp.bf16(bfloat) #0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll new file mode 100644 index 0000000..42d12fd --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll @@ -0,0 +1,94 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=SDAG-REAL16 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=SDAG-FAKE16 %s +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s + +; FIXME: GlobalISel does not work with bf16 + +declare bfloat @llvm.amdgcn.rsq.bf16(bfloat) #0 + +define amdgpu_kernel void @rsq_bf16(ptr addrspace(1) %out, bfloat %src) #1 { +; SDAG-REAL16-LABEL: rsq_bf16: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: v_rsq_bf16_e32 v0.l, s2 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: rsq_bf16: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, s2 +; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat %src) #0 + store bfloat %rsq, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_kernel void @rsq_bf16_constant_4(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: rsq_bf16_constant_4: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 4.0 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: rsq_bf16_constant_4: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, 4.0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat 4.0) #0 + store bfloat %rsq, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_kernel void @rsq_bf16_constant_100(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: rsq_bf16_constant_100: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 0x42c8 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: rsq_bf16_constant_100: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, 0x42c8 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat 100.0) #0 + store bfloat %rsq, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_kernel void @rsq_undef_bf16(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: rsq_undef_bf16: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: rsq_undef_bf16: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_endpgm + %rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat undef) + store bfloat %rsq, ptr addrspace(1) %out, align 2 + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll new file mode 100644 index 0000000..9c35a7e --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll @@ -0,0 +1,33 @@ +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s +; xUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s + +; FIXME: GlobalISel does not work with bf16 + +declare bfloat @llvm.amdgcn.sin.bf16(bfloat) #0 + +; GCN-LABEL: {{^}}sin_bf16: +; GCN: v_sin_bf16_e32 {{v[0-9]+}}, {{s[0-9]+}} +define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 { + %sin = call bfloat @llvm.amdgcn.sin.bf16(bfloat %src) #0 + store bfloat %sin, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}sin_bf16_constant_4 +; GCN: v_sin_bf16_e32 v0, 4.0 +define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 { + %sin = call bfloat @llvm.amdgcn.sin.bf16(bfloat 4.0) #0 + store bfloat %sin, ptr addrspace(1) %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}sin_bf16_constant_100 +; GCN: v_sin_bf16_e32 {{v[0-9]+}}, 0x42c8 +define amdgpu_kernel void @sin_bf16_constant_100(ptr addrspace(1) %out) #1 { + %sin = call bfloat @llvm.amdgcn.sin.bf16(bfloat 100.0) #0 + store bfloat %sin, ptr addrspace(1) %out, align 2 + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll index 344c011..dd89f80 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll @@ -1,14 +1,180 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; xUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=SDAG-REAL16 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=SDAG-REAL16 %s ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=SDAG-FAKE16 %s ; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GISEL-REAL16 %s ; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GISEL-FAKE16 %s -; FIXME: t16 doesn't work at the moment because the store of s16 under t16 mode fails to select. ; FIXME: GlobalISel does not work with bf16 +declare float @llvm.amdgcn.tanh.f32(float) #0 +declare half @llvm.amdgcn.tanh.f16(half) #0 declare bfloat @llvm.amdgcn.tanh.bf16(bfloat) #0 +define amdgpu_kernel void @tanh_f32(ptr addrspace(1) %out, float %src) #1 { +; SDAG-REAL16-LABEL: tanh_f32: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: v_tanh_f32_e32 v0, s2 +; SDAG-REAL16-NEXT: global_store_b32 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_f32: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: v_tanh_f32_e32 v0, s2 +; SDAG-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call float @llvm.amdgcn.tanh.f32(float %src) #0 + store float %tanh, ptr addrspace(1) %out, align 4 + ret void +} + +; TODO: Really these should be constant folded +define amdgpu_kernel void @tanh_f32_constant_4.0(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: tanh_f32_constant_4.0: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_tanh_f32_e32 v0, 4.0 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: global_store_b32 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_f32_constant_4.0: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_tanh_f32_e32 v0, 4.0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call float @llvm.amdgcn.tanh.f32(float 4.0) #0 + store float %tanh, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @tanh_f32_constant_100.0(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: tanh_f32_constant_100.0: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_tanh_f32_e32 v0, 0x42c80000 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: global_store_b32 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_f32_constant_100.0: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_tanh_f32_e32 v0, 0x42c80000 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call float @llvm.amdgcn.tanh.f32(float 100.0) #0 + store float %tanh, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @tanh_undef_f32(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: tanh_undef_f32: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_undef_f32: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call float @llvm.amdgcn.tanh.f32(float undef) + store float %tanh, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_kernel void @tanh_f16(ptr addrspace(1) %out, half %src) #1 { +; SDAG-REAL16-LABEL: tanh_f16: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: v_tanh_f16_e32 v0.l, s2 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_f16: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: v_tanh_f16_e32 v0, s2 +; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call half @llvm.amdgcn.tanh.f16(half %src) #0 + store half %tanh, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_kernel void @tanh_f16_constant_4.0(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: tanh_f16_constant_4.0: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_tanh_f16_e32 v0.l, 4.0 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_f16_constant_4.0: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_tanh_f16_e32 v0, 4.0 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call half @llvm.amdgcn.tanh.f16(half 4.0) #0 + store half %tanh, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_kernel void @tanh_f16_constant_100.0(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: tanh_f16_constant_100.0: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-REAL16-NEXT: v_tanh_f16_e32 v0.l, 0x5640 +; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-REAL16-NEXT: s_wait_kmcnt 0x0 +; SDAG-REAL16-NEXT: flat_store_b16 v1, v0, s[0:1] +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_f16_constant_100.0: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 +; SDAG-FAKE16-NEXT: v_tanh_f16_e32 v0, 0x5640 +; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0 +; SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; SDAG-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1] +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call half @llvm.amdgcn.tanh.f16(half 100.0) #0 + store half %tanh, ptr addrspace(1) %out, align 2 + ret void +} + +define amdgpu_kernel void @tanh_undef_f16(ptr addrspace(1) %out) #1 { +; SDAG-REAL16-LABEL: tanh_undef_f16: +; SDAG-REAL16: ; %bb.0: +; SDAG-REAL16-NEXT: s_endpgm +; +; SDAG-FAKE16-LABEL: tanh_undef_f16: +; SDAG-FAKE16: ; %bb.0: +; SDAG-FAKE16-NEXT: s_endpgm + %tanh = call half @llvm.amdgcn.tanh.f16(half undef) + store half %tanh, ptr addrspace(1) %out, align 2 + ret void +} + define amdgpu_kernel void @tanh_bf16(ptr addrspace(1) %out, bfloat %src) #1 { ; SDAG-REAL16-LABEL: tanh_bf16: ; SDAG-REAL16: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll new file mode 100644 index 0000000..52f6dab --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll @@ -0,0 +1,1010 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1200-SDAG-TRUE16 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=-real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1200-SDAG-FAKE16 +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1200-GI-TRUE16 +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=-real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1200-GI-FAKE16 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1250-SDAG-TRUE16 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1250-SDAG-FAKE16 +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1250-GI-TRUE16 +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck %s -check-prefixes=GFX1250-GI-FAKE16 + +define bfloat @v_exp2_bf16(bfloat %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.l, v0.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v0, v0 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call bfloat @llvm.exp2.bf16(bfloat %in) + ret bfloat %result +} + +define bfloat @v_exp2_fabs_bf16(bfloat %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_fabs_bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fabs_bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_fabs_bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.l, |v0.l| +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_fabs_bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, |v0| +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call bfloat @llvm.fabs.bf16(bfloat %in) + %result = call bfloat @llvm.exp2.bf16(bfloat %fabs) + ret bfloat %result +} + +define bfloat @v_exp2_fneg_fabs_bf16(bfloat %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_fneg_fabs_bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v1.h, 0x8000, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fneg_fabs_bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, 0x8000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_fneg_fabs_bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.l, -|v0.l| +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_fneg_fabs_bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, -|v0| +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call bfloat @llvm.fabs.bf16(bfloat %in) + %fneg.fabs = fneg bfloat %fabs + %result = call bfloat @llvm.exp2.bf16(bfloat %fneg.fabs) + ret bfloat %result +} + +define bfloat @v_exp2_fneg_bf16(bfloat %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_fneg_bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fneg_bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_fneg_bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.l, -v0.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_fneg_bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, -v0 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fneg = fneg bfloat %in + %result = call bfloat @llvm.exp2.bf16(bfloat %fneg) + ret bfloat %result +} + +define bfloat @v_exp2_bf16_fast(bfloat %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_bf16_fast: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_bf16_fast: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_bf16_fast: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.l, v0.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_bf16_fast: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v0, v0 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call fast bfloat @llvm.exp2.bf16(bfloat %in) + ret bfloat %result +} + +define <2 x bfloat> @v_exp2_v2bf16(<2 x bfloat> %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_v2bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_v2bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_v2bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.h, v0.h +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.l, v0.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_v2bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v0, v0 +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v1, v1 +; GFX1250-SDAG-FAKE16-NEXT: v_nop +; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call <2 x bfloat> @llvm.exp2.v2bf16(<2 x bfloat> %in) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_exp2_fabs_v2bf16(<2 x bfloat> %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_fabs_v2bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h +; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fabs_v2bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v1, v1, v3 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_fabs_v2bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX1250-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 15 +; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.l, v1.l +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.h, v2.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_fabs_v2bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX1250-SDAG-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 15 +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v1, v1 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v0, v0 +; GFX1250-SDAG-FAKE16-NEXT: v_nop +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in) + %result = call <2 x bfloat> @llvm.exp2.v2bf16(<2 x bfloat> %fabs) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_exp2_fneg_fabs_v2bf16(<2 x bfloat> %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_fneg_fabs_v2bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 15 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, 0x8000, v1.l +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v2 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v2, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, 0x8000, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v2 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v2, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, s0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fneg_fabs_v2bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 15 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX1200-SDAG-FAKE16-NEXT: v_xor_b32_e32 v1, 0x8000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_fneg_fabs_v2bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX1250-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 15 +; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.l, -v1.l +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.h, -v2.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_fneg_fabs_v2bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX1250-SDAG-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 15 +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v1, -v1 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, -v0 +; GFX1250-SDAG-FAKE16-NEXT: v_nop +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in) + %fneg.fabs = fneg <2 x bfloat> %fabs + %result = call <2 x bfloat> @llvm.exp2.v2bf16(<2 x bfloat> %fneg.fabs) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_exp2_fneg_v2bf16(<2 x bfloat> %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_fneg_v2bf16: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v2, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, s0 +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3 +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1 +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fneg_v2bf16: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_xor_b32_e32 v1, 0x8000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v1, v1, v3 +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_fneg_v2bf16: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.h, -v0.h +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e64 v0.l, -v0.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_fneg_v2bf16: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v0, -v0 +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e64 v1, -v1 +; GFX1250-SDAG-FAKE16-NEXT: v_nop +; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fneg = fneg <2 x bfloat> %in + %result = call <2 x bfloat> @llvm.exp2.v2bf16(<2 x bfloat> %fneg) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_exp2_v2bf16_fast(<2 x bfloat> %in) { +; GFX1200-SDAG-TRUE16-LABEL: v_exp2_v2bf16_fast: +; GFX1200-SDAG-TRUE16: ; %bb.0: +; GFX1200-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX1200-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo +; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h +; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0 +; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1200-SDAG-FAKE16-LABEL: v_exp2_v2bf16_fast: +; GFX1200-SDAG-FAKE16: ; %bb.0: +; GFX1200-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1200-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; GFX1200-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xf1ff +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0 +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_exp_f32_e32 v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: v_ldexp_f32 v1, v1, v2 +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX1200-SDAG-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1 +; GFX1200-SDAG-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 +; GFX1200-SDAG-FAKE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0 +; GFX1200-SDAG-FAKE16-NEXT: s_wait_alu 0xfffd +; GFX1200-SDAG-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo +; GFX1200-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302 +; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1250-SDAG-TRUE16-LABEL: v_exp2_v2bf16_fast: +; GFX1250-SDAG-TRUE16: ; %bb.0: +; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.h, v0.h +; GFX1250-SDAG-TRUE16-NEXT: v_exp_bf16_e32 v0.l, v0.l +; GFX1250-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX1250-SDAG-FAKE16-LABEL: v_exp2_v2bf16_fast: +; GFX1250-SDAG-FAKE16: ; %bb.0: +; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX1250-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v0, v0 +; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1250-SDAG-FAKE16-NEXT: v_exp_bf16_e32 v1, v1 +; GFX1250-SDAG-FAKE16-NEXT: v_nop +; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX1250-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call fast <2 x bfloat> @llvm.exp2.v2bf16(<2 x bfloat> %in) + ret <2 x bfloat> %result +} + +declare bfloat @llvm.exp2.bf16(bfloat) #0 +declare <2 x bfloat> @llvm.exp2.v2bf16(<2 x bfloat>) #0 +declare bfloat @llvm.fabs.bf16(bfloat) #0 +declare <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat>) #0 + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll index 0d5846a..5634df5 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll @@ -6377,28 +6377,99 @@ define float @v_log_f32_from_fpext_bf16(bfloat %src) { ; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX900-NEXT: s_setpc_b64 s[30:31] ; -; GFX1100-LABEL: v_log_f32_from_fpext_bf16: -; GFX1100: ; %bb.0: -; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo -; GFX1100-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX1100-NEXT: v_log_f32_e32 v0, v0 -; GFX1100-NEXT: s_waitcnt_depctr 0xfff -; GFX1100-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0 -; GFX1100-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| -; GFX1100-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2 -; GFX1100-NEXT: v_add_f32_e32 v1, v1, v2 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 -; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo -; GFX1100-NEXT: v_sub_f32_e32 v0, v0, v1 -; GFX1100-NEXT: s_setpc_b64 s[30:31] +; GFX1100-SDAG-TRUE16-LABEL: v_log_f32_from_fpext_bf16: +; GFX1100-SDAG-TRUE16: ; %bb.0: +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-SDAG-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0 +; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-SDAG-TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2 +; GFX1100-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-SDAG-FAKE16-LABEL: v_log_f32_from_fpext_bf16: +; GFX1100-SDAG-FAKE16: ; %bb.0: +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-SDAG-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0 +; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-SDAG-FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2 +; GFX1100-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-GISEL-TRUE16-LABEL: v_log_f32_from_fpext_bf16: +; GFX1100-GISEL-TRUE16: ; %bb.0: +; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX1100-GISEL-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-GISEL-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-GISEL-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0 +; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-GISEL-TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1 +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2 +; GFX1100-GISEL-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo +; GFX1100-GISEL-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-GISEL-FAKE16-LABEL: v_log_f32_from_fpext_bf16: +; GFX1100-GISEL-FAKE16: ; %bb.0: +; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo +; GFX1100-GISEL-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0 +; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2 +; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo +; GFX1100-GISEL-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; R600-LABEL: v_log_f32_from_fpext_bf16: ; R600: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll index 8006876..8d1a231 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll @@ -6377,28 +6377,99 @@ define float @v_log10_f32_from_fpext_bf16(bfloat %src) { ; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX900-NEXT: s_setpc_b64 s[30:31] ; -; GFX1100-LABEL: v_log10_f32_from_fpext_bf16: -; GFX1100: ; %bb.0: -; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo -; GFX1100-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX1100-NEXT: v_log_f32_e32 v0, v0 -; GFX1100-NEXT: s_waitcnt_depctr 0xfff -; GFX1100-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0 -; GFX1100-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| -; GFX1100-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2 -; GFX1100-NEXT: v_add_f32_e32 v1, v1, v2 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1100-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 -; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo -; GFX1100-NEXT: v_sub_f32_e32 v0, v0, v1 -; GFX1100-NEXT: s_setpc_b64 s[30:31] +; GFX1100-SDAG-TRUE16-LABEL: v_log10_f32_from_fpext_bf16: +; GFX1100-SDAG-TRUE16: ; %bb.0: +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-SDAG-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0 +; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-SDAG-TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2 +; GFX1100-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-SDAG-FAKE16-LABEL: v_log10_f32_from_fpext_bf16: +; GFX1100-SDAG-FAKE16: ; %bb.0: +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-SDAG-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0 +; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-SDAG-FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2 +; GFX1100-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-GISEL-TRUE16-LABEL: v_log10_f32_from_fpext_bf16: +; GFX1100-GISEL-TRUE16: ; %bb.0: +; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX1100-GISEL-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-GISEL-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-GISEL-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0 +; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-GISEL-TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1 +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2 +; GFX1100-GISEL-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo +; GFX1100-GISEL-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-GISEL-FAKE16-LABEL: v_log10_f32_from_fpext_bf16: +; GFX1100-GISEL-FAKE16: ; %bb.0: +; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo +; GFX1100-GISEL-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0 +; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0| +; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2 +; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0 +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo +; GFX1100-GISEL-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; R600-LABEL: v_log10_f32_from_fpext_bf16: ; R600: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.bf16.ll new file mode 100644 index 0000000..5bd9fa6 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.bf16.ll @@ -0,0 +1,240 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GFX-SDAG-TRUE16 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GFX-SDAG-FAKE16 %s +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefix=GFX-GISEL-TRUE16 %s +; xUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefix=GFX-GISEL-FAKE16 %s + +define bfloat @v_log2_bf16(bfloat %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.l, v0.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v0, v0 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call bfloat @llvm.log2.bf16(bfloat %in) + ret bfloat %result +} + +define bfloat @v_log2_fabs_bf16(bfloat %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_fabs_bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.l, |v0.l| +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_fabs_bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, |v0| +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call bfloat @llvm.fabs.bf16(bfloat %in) + %result = call bfloat @llvm.log2.bf16(bfloat %fabs) + ret bfloat %result +} + +define bfloat @v_log2_fneg_fabs_bf16(bfloat %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_fneg_fabs_bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.l, -|v0.l| +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_fneg_fabs_bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, -|v0| +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call bfloat @llvm.fabs.bf16(bfloat %in) + %fneg.fabs = fneg bfloat %fabs + %result = call bfloat @llvm.log2.bf16(bfloat %fneg.fabs) + ret bfloat %result +} + +define bfloat @v_log2_fneg_bf16(bfloat %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_fneg_bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.l, -v0.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_fneg_bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, -v0 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fneg = fneg bfloat %in + %result = call bfloat @llvm.log2.bf16(bfloat %fneg) + ret bfloat %result +} + +define bfloat @v_log2_bf16_fast(bfloat %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_bf16_fast: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.l, v0.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_bf16_fast: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v0, v0 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call fast bfloat @llvm.log2.bf16(bfloat %in) + ret bfloat %result +} + +define <2 x bfloat> @v_log2_v2bf16(<2 x bfloat> %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_v2bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.h, v0.h +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.l, v0.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_v2bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v0, v0 +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v1, v1 +; GFX-SDAG-FAKE16-NEXT: v_nop +; GFX-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %in) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_log2_fabs_v2bf16(<2 x bfloat> %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_fabs_v2bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 15 +; GFX-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.l, v1.l +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.h, v2.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_fabs_v2bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX-SDAG-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 15 +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v1, v1 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v0, v0 +; GFX-SDAG-FAKE16-NEXT: v_nop +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in) + %result = call <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %fabs) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_log2_fneg_fabs_v2bf16(<2 x bfloat> %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_fneg_fabs_v2bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 15 +; GFX-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.l, -v1.l +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.h, -v2.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_fneg_fabs_v2bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX-SDAG-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 15 +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v1, -v1 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, -v0 +; GFX-SDAG-FAKE16-NEXT: v_nop +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in) + %fneg.fabs = fneg <2 x bfloat> %fabs + %result = call <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %fneg.fabs) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_log2_fneg_v2bf16(<2 x bfloat> %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_fneg_v2bf16: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.h, -v0.h +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e64 v0.l, -v0.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_fneg_v2bf16: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v0, -v0 +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e64 v1, -v1 +; GFX-SDAG-FAKE16-NEXT: v_nop +; GFX-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %fneg = fneg <2 x bfloat> %in + %result = call <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %fneg) + ret <2 x bfloat> %result +} + +define <2 x bfloat> @v_log2_v2bf16_fast(<2 x bfloat> %in) { +; GFX-SDAG-TRUE16-LABEL: v_log2_v2bf16_fast: +; GFX-SDAG-TRUE16: ; %bb.0: +; GFX-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.h, v0.h +; GFX-SDAG-TRUE16-NEXT: v_log_bf16_e32 v0.l, v0.l +; GFX-SDAG-TRUE16-NEXT: s_set_pc_i64 s[30:31] +; +; GFX-SDAG-FAKE16-LABEL: v_log2_v2bf16_fast: +; GFX-SDAG-FAKE16: ; %bb.0: +; GFX-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v0, v0 +; GFX-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX-SDAG-FAKE16-NEXT: v_log_bf16_e32 v1, v1 +; GFX-SDAG-FAKE16-NEXT: v_nop +; GFX-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31] + %result = call fast <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %in) + ret <2 x bfloat> %result +} + +declare bfloat @llvm.log2.bf16(bfloat) #0 +declare <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat>) #0 +declare bfloat @llvm.fabs.bf16(bfloat) #0 +declare <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat>) #0 + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll index c1ac74e..7ca72bf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll @@ -3830,20 +3830,67 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) { ; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1 ; GFX900-NEXT: s_setpc_b64 s[30:31] ; -; GFX1100-LABEL: v_log2_f32_from_fpext_bf16: -; GFX1100: ; %bb.0: -; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1100-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 -; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo -; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo -; GFX1100-NEXT: v_ldexp_f32 v0, v0, v2 -; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1100-NEXT: v_log_f32_e32 v0, v0 -; GFX1100-NEXT: s_waitcnt_depctr 0xfff -; GFX1100-NEXT: v_sub_f32_e32 v0, v0, v1 -; GFX1100-NEXT: s_setpc_b64 s[30:31] +; GFX1100-SDAG-TRUE16-LABEL: v_log2_f32_from_fpext_bf16: +; GFX1100-SDAG-TRUE16: ; %bb.0: +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 +; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-SDAG-FAKE16-LABEL: v_log2_f32_from_fpext_bf16: +; GFX1100-SDAG-FAKE16: ; %bb.0: +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-GISEL-TRUE16-LABEL: v_log2_f32_from_fpext_bf16: +; GFX1100-GISEL-TRUE16: ; %bb.0: +; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0 +; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1 +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo +; GFX1100-GISEL-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0 +; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1100-GISEL-TRUE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-GISEL-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX1100-GISEL-FAKE16-LABEL: v_log2_f32_from_fpext_bf16: +; GFX1100-GISEL-FAKE16: ; %bb.0: +; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1100-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0 +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo +; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo +; GFX1100-GISEL-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0 +; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr 0xfff +; GFX1100-GISEL-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1 +; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; R600-LABEL: v_log2_f32_from_fpext_bf16: ; R600: ; %bb.0: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.rint.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.rint.f64.ll index 28781ae..c6cf6f6 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.rint.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.rint.f64.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s ; RUN: llc -mtriple=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s ; RUN: llc -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll index af914bd..355f77a 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll @@ -76,12 +76,13 @@ define amdgpu_kernel void @v_round_f64(ptr addrspace(1) %out, ptr addrspace(1) % ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_mov_b64 s[4:5], s[2:3] ; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 +; SI-NEXT: s_movk_i32 s4, 0xfc01 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_mov_b32 s3, 0xfffff ; SI-NEXT: v_mov_b32_e32 v8, 0x3ff00000 ; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_bfe_u32 v4, v3, 20, 11 -; SI-NEXT: v_add_i32_e32 v6, vcc, 0xfffffc01, v4 +; SI-NEXT: v_add_i32_e32 v6, vcc, s4, v4 ; SI-NEXT: v_lshr_b64 v[4:5], s[2:3], v6 ; SI-NEXT: v_and_b32_e32 v7, 0x80000000, v3 ; SI-NEXT: v_not_b32_e32 v5, v5 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll index 47b2b68..dcf01f7 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll @@ -2,8 +2,6 @@ ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-TRUE16 %s ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX12-FAKE16 %s -; FIXME: t16 doesn't work at the moment because the store of s16 under t16 mode fails to select. - declare bfloat @llvm.sqrt.bf16(bfloat %a) declare <2 x bfloat> @llvm.sqrt.v2bf16(<2 x bfloat> %a) diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll index 6e94896..c0fb145 100644 --- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll +++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll @@ -193,13 +193,22 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src } define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, half %src2) #0 { -; SDAG-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack: -; SDAG-GFX11: ; %bb.0: -; SDAG-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] -; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; SDAG-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; SDAG-GFX11-NEXT: s_setpc_b64 s[30:31] +; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack: +; SDAG-GFX11-TRUE16: ; %bb.0: +; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v1, v0, v1, v2 op_sel_hi:[1,1,1] +; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l +; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack: +; SDAG-GFX11-FAKE16: ; %bb.0: +; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] +; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; SDAG-GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack: ; SDAG-GFX9: ; %bb.0: @@ -265,13 +274,22 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, ha } define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src1, half %src2) #0 { -; SDAG-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext: -; SDAG-GFX11: ; %bb.0: -; SDAG-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SDAG-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] -; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; SDAG-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; SDAG-GFX11-NEXT: s_setpc_b64 s[30:31] +; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext: +; SDAG-GFX11-TRUE16: ; %bb.0: +; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v1, v0, v1, v2 op_sel_hi:[1,1,1] +; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0 +; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l +; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext: +; SDAG-GFX11-FAKE16: ; %bb.0: +; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SDAG-GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] +; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; SDAG-GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext: ; SDAG-GFX9: ; %bb.0: @@ -569,3 +587,4 @@ attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" attributes #1 = { nounwind readnone speculatable } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GISEL-GFX11-FAKE16: {{.*}} +; SDAG-GFX11: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll index 6246f2f..ca16e25 100644 --- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll @@ -118,34 +118,29 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) { ; GFX11-TRUE16-LABEL: v_maximumnum_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_bf16: @@ -181,40 +176,34 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_bf16: @@ -339,21 +328,21 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) { ; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_bf16_nnan: @@ -381,25 +370,25 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_bf16_nnan: @@ -630,58 +619,46 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v3, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16: @@ -738,62 +715,56 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v3, v4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16: @@ -1012,34 +983,29 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y) ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h ; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, s2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16_nnan: @@ -1085,36 +1051,35 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y) ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h ; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.h, v0.l, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v0.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, s1 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16_nnan: @@ -1444,66 +1409,67 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v7, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v9, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16: @@ -1575,77 +1541,80 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v7, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v9, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16: @@ -1939,41 +1908,40 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y) ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v7, v6 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v9, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v0.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v0.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v2.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_nnan: @@ -2029,48 +1997,50 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y) ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v7, v6 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v9, v8 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v0.h, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v0.l, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_nnan: @@ -2507,85 +2477,83 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v10, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v9, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v8, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16: @@ -2680,99 +2648,98 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v10, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v9, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v7.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v8.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v8, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v5.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16: @@ -3158,53 +3125,52 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y) ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v7, v6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v1.h, s4 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v6, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v3.h -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.h, v0.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_nnan: @@ -3273,62 +3239,63 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y) ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l ; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v7, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v1.h, s4 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v6, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v1.l, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v3.h -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.h, v0.l, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v0.l, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_nnan: @@ -3957,125 +3924,120 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v2.h, v5.h, vcc_lo ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v13, v13 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v8 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v9.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v4.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v10.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v9.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v7.l ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v8.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v3.h, s2 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v13, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v7.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v9.l, v8.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v13.l, v8.l, s3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v16 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v12.l, v10.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v10.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v7 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v12.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v4.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 ; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v3.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v0.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v11.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v9.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v8.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.l, v5.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v5.l, v12.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v12.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v12, v12 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v4.l, v1.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v11.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v4.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v1, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.h, v1.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v5, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.h, v10.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v9.h, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v9.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v1.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.l, v0.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v0, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v9.h, v0.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v8.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v9.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v8.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v6.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s4 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v6bf16: @@ -4206,142 +4168,141 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v9.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v2.h, v5.h, vcc_lo ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v13, v13 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v8 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v8.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v3.h, s2 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v13, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v9.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v7.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v4.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v9.l, v8.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v13.l, v8.l, s3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v4.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v10.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v12.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v9.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v7 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v7.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v0.h, v3.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v12.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v11.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v9.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v4.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v8.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.l, v5.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v5.l, v12.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v12.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v12.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v5, v5 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v12, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v4.l, v1.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v11.h, s7 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v1, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.h, v1.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v9.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v5, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.h, v10.h, s6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v9.h, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v9.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v1.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.l, v0.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v0, v9 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v9.h, v0.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v8.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v9.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v8.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v6.l, v0.h, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v3.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s4 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v6bf16: @@ -5219,171 +5180,160 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v3.h, v7.h, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v15, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v5.h, v12.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v13, v18 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v10.l, v8.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v11.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v9.l, s3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v13.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v15.l, v9.l, s4 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v17, v17 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v17, v17 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v8.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v6.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v13.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v13.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v9.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v1.h, v5.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v5.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v14.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v0.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v4.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v15.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v11.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.l, v7.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v7.l, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v16.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v12.h, v16.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v7, v7 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v16, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v6.l, v2.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v2.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v10.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v6.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v2, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v12.h, v2.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v12.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v7, v7 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v13.h, s8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.h, v12.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, s2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.h, v11.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v15.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.h, v10.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v14.l, v12.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v14.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v8.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v8.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v7.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v15, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.l, v2.l, s3 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v6.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v3.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.l, v4.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v1.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v5.l, v1.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v1, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v12.h, v1.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v12.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v7, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v12.h, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v12.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v8 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.l, v0.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s6, v0, v12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v12.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v12.h, v0.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v11.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.h, v11.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v14.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v15.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v12.h, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.h, v14.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v15.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v1.h, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v0.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v5.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v8bf16: @@ -5546,201 +5496,187 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v3.h, v7.h, vcc_lo +; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v15, v17 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v5.h, v12.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v13, v18 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v10.l, v8.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v11.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v9.l, s3 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v13.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v15.l, v9.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.h, v11.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v17, v17 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v17, v17 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v15.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.h, v10.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v14.l, v12.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v12.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v9.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v8.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v14.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.h, v6.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v6.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v13.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v8.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v8.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v8.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v7.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v13.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v9.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v1.h, v5.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v5.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v14 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v14.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v0.h, v4.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v15, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v4.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v15 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v15.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.l, v2.l, s3 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v11.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.l, v7.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v7.l, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v16.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v12.h, v16.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v7, v7 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v16, v16 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v6.l, v2.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v2.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v2, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v12.h, v2.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v7, v7 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v13.h, s8 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.h, v12.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v5.l, v1.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v1.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v6.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v1, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v12.h, v1.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v12.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v7, v7 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v12.h, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v12.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.l, v0.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s6, v0, v12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v12.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v3.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v12.h, v0.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v11.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.l, v4.h, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v1.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.h, v11.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v14.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v15.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v12.h, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v0.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.h, v14.h, s6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v15.h, s7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v1.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v0.h, s3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v5.l, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v8bf16: @@ -7352,341 +7288,314 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) { ; GFX11-TRUE16-LABEL: v_maximumnum_v16bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v18, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v17 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v13 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v19 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v18 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v13.h, v20.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v24, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v19.l, v7.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v19.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v23.l, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v20.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v19.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v6.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v11 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v3.h, v11.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v26, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v19.l, v22.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v11.h, v20.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v7.l, v5.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v7.h, v15.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v16.l +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v15.h, v16.h, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v17 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v19.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v16.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.h, v16.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v21.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v20.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v7.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v22.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v1.h, v9.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v9.h, v23.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v20.l, v21.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v27, v25 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v7.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v7.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v5.l, v23.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v23.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v21.l, v23.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v20.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.l, v16.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v7.l, v19.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v14.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v19.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v24, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v6.l, v16.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.l, v16.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v14.l, v17.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v16.l, v17.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v14.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v14.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v15.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v17.l, v16.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v6.h, v14.h, s0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v8 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v29, v29 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v14.h, v16.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v32, v32 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v18 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.h, v16.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v18.l, v16.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v18.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v5.h, v13.h, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.h, v13.h, v16.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v16.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v25 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v25.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v32, v32 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v25.h, v16.h, s1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v16.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v20.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v32, v32 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v16.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v4.h, v12.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.h, v12.h, v16.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v26.h, v16.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.l, v16.h, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v21.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.h, v11.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v16, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v27.h, v16.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.l, v16.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v22.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v2.h, v10.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v10.h, v16.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v16, v28 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v28.h, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.l, v16.h, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v23.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v1.h, v9.h, s5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v9.h, v16.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v16, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v29.h, v16.h, s5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.l, v16.h, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v24.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v30, v30 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v0.h, v8.h, s6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v8.h, v16.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s6, v16, v30 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v30.h, v16.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v15.h, v16.h, s7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v15.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v15.l, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v16.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v7.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v7.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s7, v7, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v16.h, v7.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v6.h, v16.h, s7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v7.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, s8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v14.l, v6.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v14.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s8, v6, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v6.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v5.h, v16.h, s8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v6.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v13.l, v5.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v13.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s9, v5, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v16.h, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, s10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v12.l, v4.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v12.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s10, v4, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v16.h, v4.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v16.h, s10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, s11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v11.l, v3.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v11.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s11, v3, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.h, v3.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v3.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.h, v16.h, s11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, s12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v10.l, v2.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v10.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s12, v2, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v16.h, v2.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v16.h, s12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, s13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v9.l, v1.h, s14 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v9.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s13, v1, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v0.l, v8.l, s14 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v16.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v1.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v17.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v1.h, s15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.l, v17.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v18.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v20.h, v25.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v32.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.h, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.h, v18.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v26.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v20.l, v1.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v8.l, v32.h, s14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v18.l, v0.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.h, v26.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v28.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v27.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v32, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v21.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v29.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v23.h, v28.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.h, v32.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v22.h, v27.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v29.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v30.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.l, v32.h, s15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v23.l, v1.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v22.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v8.h, s5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v25.l, v30.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v16.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v8.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.h, v0.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v12.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.l, v4.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v17, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v10.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v8.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v12.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v22, v17 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v11.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v2.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.l, v1.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v10.l, v0.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v13.l, v2.h, s3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v19 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v8.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v16bf16: @@ -8005,406 +7914,355 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v18, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v15 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v17 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v13 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v5.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v19 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v13.h, v20.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v24, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v19.l, v7.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v19.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v23.l, v7.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v25 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v20.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v19.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v6.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v20 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v3.h, v11.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v26, v21 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v19.l, v22.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v11.h, v20.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v7.l, v5.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v13 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v7.h, v15.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v15.h, v16.h, vcc_lo +; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v16.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v3 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v17 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l ; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v19.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v23 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v16.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v25 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.h, v16.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v21.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v20.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v7.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v22.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v1.h, v9.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v21.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v9.h, v23.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v20.l, v21.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v27, v25 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo -; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v7.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v7.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v5.l, v23.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v23.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v21.l, v23.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v20.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.l, v16.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v7.l, v19.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v19.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v14.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v19.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v24, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v7.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v6.l, v16.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.l, v16.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v14.l, v17.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v16.l, v17.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v14.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v17.l, v16.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v6.h, v14.h, s0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v8 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v29, v29 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v14.h, v16.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v7 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v32, v32 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v18 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v14 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v14.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.h, v16.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v18.l, v16.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v18.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v5.h, v13.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.h, v13.h, v16.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v25 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v25.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v32, v32 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v15.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v25.h, v16.h, s1 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v16.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v20.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v32, v32 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v32.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v4.h, v12.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.h, v12.h, v16.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v26 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v12.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.l, v4.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v17, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v26.h, v16.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.l, v16.h, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v21.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v10.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.h, v11.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v16, v27 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v12.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v27.h, v16.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.l, v16.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v22.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v8.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v12.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v22, v17 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v2.h, v10.h, s4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v10.h, v16.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v16, v28 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v28.h, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.l, v16.h, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v23.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v1.h, v9.h, s5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v9.h, v16.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v16, v29 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v9.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v29.h, v16.h, s5 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.l, v16.h, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v24.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v30, v30 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v0.h, v8.h, s6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v8.h, v16.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s6, v16, v30 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v30.h, v16.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v15.h, v16.h, s7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v15.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v15.l, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v7.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v7.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s7, v7, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v16.h, v7.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v16.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v6.h, v16.h, s7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, s8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v14.l, v6.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v6.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v11.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v14.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s8, v6, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v6.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v16.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v5.h, v16.h, s8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v6.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v13.l, v5.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v5.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v13.h, s8 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s9, v5, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v16.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v16.h, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, s10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v12.l, v4.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v4.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v12.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s10, v4, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v16.h, v4.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v16.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v16.h, s10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v4.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, s11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v11.l, v3.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v3.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v11.h, s10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s11, v3, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.h, v3.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v16.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v3.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.h, v16.h, s11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v3.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, s12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v10.l, v2.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v2.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v2.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.l, v1.h, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v10.l, v0.h, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v13.l, v2.h, s3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v10.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s12, v2, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v16.h, v2.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v16.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v16.h, s12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, s13 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v16 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v19 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v9.l, v1.h, s14 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v1.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v8 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v9.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s13, v1, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v0.l, v8.l, s14 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v16.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v1.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v17.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v1.h, s15 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.l, v17.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v18.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v20.h, v25.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v32.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.h, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v1.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.h, v18.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v26.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v20.l, v1.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v8.l, v32.h, s14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v18.l, v0.h, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.h, v26.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v28.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v27.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v32, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v21.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v29.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v23.h, v28.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.h, v32.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v22.h, v27.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v29.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v30.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.l, v32.h, s15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v23.l, v1.h, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v22.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v8.h, s5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v25.l, v30.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v16.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v8.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.h, v0.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v8.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v16bf16: @@ -11681,666 +11539,619 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) { ; GFX11-TRUE16-LABEL: v_maximumnum_v32bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: scratch_load_b32 v68, off, s32 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v29 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v36.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v36.l ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v13 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v36.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v12 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v28 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v27 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v55.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v86, v118 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v85.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v64.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v65.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v66.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v67.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v68.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v69.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v70.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v71.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v80.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v81.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v82.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v83.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v84.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v119 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v128 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v130 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v131 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v132 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v133 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v134 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v135 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v144 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v145 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s63, v116, v86 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v86.l, v55.l, v32.l, s40 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v55.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v146 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v147 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v87, v118 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v96, v119 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s45, v98, v129 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s56, v101, v132 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s59, v112, v135 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s60, v113, v144 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v86.l, v32.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v86.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v34.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v36.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v39.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v50.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v51.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s46, v99, v130 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s61, v114, v145 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s62, v115, v146 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.l, v65.l, v34.l, s43 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v98.l, v67.l, v36.l, s45 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v101.l, v70.l, v39.l, s56 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v112.l, v81.l, v50.l, s59 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v113.l, v82.l, v51.l, s60 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.h, v55.l, s16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v118 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v33.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v37.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v52.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0, v70.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0, v81.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v64.l, v33.l, s42 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v99.l, v68.l, v37.l, s46 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v114.l, v83.l, v52.l, s61 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v115.l, v84.l, v53.l, s62 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v96.l, v34.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v98.l, v36.l, s5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v101.l, v39.l, s8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v101.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v112.l, v50.l, s11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v112.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v113.l, v51.l, s12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v113.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v54.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v97, v128 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v116.l, v85.l, v54.l, s63 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v87.l, v33.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v99.l, v37.l, s6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v114.l, v52.l, s13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v114.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v115.l, v53.l, s14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v115.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v70.l, s23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v39 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v35.h, v81.l, s26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v35.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s57, v102, v133 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v97.l, v66.l, v35.l, s44 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v116.l, v54.l, s15 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v51 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v48.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v65.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v66.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s47, v100, v131 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s58, v103, v134 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v71.l, v48.l, s57 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v96.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v97.l, v35.l, s4 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v97.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v52 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v38.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v49.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v64.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0, v71.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v100.l, v69.l, v38.l, s47 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v103.l, v80.l, v49.l, s58 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v87.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v102.l, v48.l, s9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v102.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v30.h, v65.l, s18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v128 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v66.l, s19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v129 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0, v80.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0, v85.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v38.l, s7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v103.l, v49.l, s10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v103.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v14.h, v64.l, s17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v119 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v34.h, v71.l, s24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v48 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v65 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v66 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v86.l, v13.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v80.l, s25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v49 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v37.h, v85.l, s41 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v64 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v71 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v96.l, v30.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v97.l, v32.l, s4 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0, v67.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0, v68.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v98.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v99.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v87.l, v38.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v102.l, v38.h, s9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v67.l, s20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v130 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v68.l, s21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v131 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0, v84.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v70 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v67 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0, v82.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v68 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v84.l, s29 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v101.l, v34.l, s8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0, v83.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v80 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v82.l, s27 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v115.l, v37.l, s14 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0, v69.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v36.h, v83.l, s28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v103.l, v35.l, s10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v113.l, v36.l, s12 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v100.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.h, v69.l, s22 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v81 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.h, v114.l, v48.l, s13 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v132 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v69 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v33.h, s7 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v39, v39 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v9 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v8 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v23 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v22 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v21 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v19 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v18 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v17 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v15 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v30 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v17 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v16 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v83, v83 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v86, v86 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.h, v0.l, v16.l, s42 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s43, 0, v96.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v31 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v68 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v15.h, v68.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v68.h, v36.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v36, v35 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0, v35.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v35.h, v36.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v31.l, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v31.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v14.h, v30.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v30.h, v36.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v36, v37 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s44, 0, v37.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v37.h, v36.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.l, v36.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v32.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v13.h, v29.h, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v29.h, v36.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v36, v38 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v38.h, v36.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.l, v36.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v33.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v12.h, v28.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v28.h, v36.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v36, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v39.h, v36.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v34.l, v36.h, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v34.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v48, v48 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v36.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v50.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v15.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v98.l, v32.h, s5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v51.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v99.l, v33.l, s6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v54, v54 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v50.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v52, v53 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v11.h, v27.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.h, v27.h, v36.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v36, v48 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v48.h, v36.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v35.l, v36.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v35.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v36.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v51.l, v50.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v52 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v51.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v32.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v50.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v10.h, v26.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v26.h, v36.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v36, v49 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v49.h, v36.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v38.l, v36.h, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v38.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v9.h, v25.h, s5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.h, v25.h, v36.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v36, v50 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v50.h, v36.h, s5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v48.l, v36.h, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v48.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v8.h, v24.h, s6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.h, v24.h, v36.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s6, v36, v51 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v51.h, v36.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v50.l, v36.h, s7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v50.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v7.h, v23.h, s7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v23.h, v36.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s7, v36, v52 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.h, v36.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v52.l, v36.h, s8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v52.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v6.h, v22.h, s8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.h, v22.h, v36.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s8, v36, v53 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.h, v36.h, s8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.h, v53.l, v36.h, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v53.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v5.h, v21.h, s9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.h, v36.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s9, v36, v54 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.h, v36.h, s9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v54.l, v36.h, s10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v54.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v4.h, v20.h, s10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.h, v20.h, v36.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s10, v36, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.h, v36.h, s10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.h, v55.l, v36.h, s11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v55.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v3.h, v19.h, s11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.h, v19.h, v36.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s11, v36, v64 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.h, v36.h, s11 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v64.l, v36.h, s12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v64.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v2.h, v18.h, s12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.h, v18.h, v36.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s12, v36, v65 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.h, v36.h, s12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.h, v65.l, v36.h, s13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v65.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v1.h, v17.h, s13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.h, v17.h, v36.h, s14 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v36.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s13, v36, v66 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.h, v36.h, s13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v66.l, v36.h, s14 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v66.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v0.h, v16.h, s14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.h, v16.h, v36.h, s15 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v68 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s14, v36, v67 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.h, v15.l, v68.l, s15 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.h, v36.h, s14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v67.l, v36.h, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v67.l +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v82.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v68.l, v82.h, s15 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s15, v82, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v36.h, v82.h, s15 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v68.l, v82.h, s16 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v13 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v14.h, v36.h, s15 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v68.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v14.l, v30.l, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s15, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v30.l, v14.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v14.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v38, v53 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v112.l, v39.l, s11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v15.h, v51.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v31.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v29 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v50 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v30.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v33.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v15.h, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v117, v117 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v31.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v27 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s16, v14, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v82, v82 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v36.h, v14.h, s16 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v14.l, v14.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v12 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v25 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v13.h, v36.h, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v14.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.l, v29.l, s17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v24 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s16, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v29.l, v13.h, s18 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v23 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s17, v13, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v29.h, s16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v82, v82 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v36.h, v13.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v13.l, v13.h, s18 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v11 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v21 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v12.h, v36.h, s17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v13.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.l, v28.l, s18 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v20 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v28.l, v12.h, s19 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v12.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v19 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v12, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v28.h, s17 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v82, v82 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v36.h, v12.h, s18 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v36.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v12.l, v12.h, s19 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v36.h, s18 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v12.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.l, v27.l, s19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v27.l, v11.h, s20 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0, v11.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v27.h, s18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v11, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v36.h, v11.h, s19 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v11.l, v11.h, s20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.h, v10.h, v36.h, s19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v26.l, s20 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v26.l, v10.h, s21 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0, v10.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v26.h, s19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v10, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v36.h, v10.h, s20 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v10.l, v10.h, s21 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.h, v9.h, v36.h, s20 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v25.l, s21 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v25.l, v9.h, s22 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0, v9.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v25.h, s20 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v9, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v36.h, v9.h, s21 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v9.h, s22 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v36.h, s21 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v24.l, s22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v24.l, v8.h, s23 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0, v8.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v24.h, s21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v8, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v36.h, v8.h, s22 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v8.l, v8.h, s23 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v7.h, v36.h, s22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v8.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, s23 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v23.l, v7.h, s24 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v23.h, s22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v7, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v36.h, v7.h, s23 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s24 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v6.h, v36.h, s23 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, s24 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v22.l, v6.h, s25 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v22.h, s23 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v6, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v36.h, v6.h, s24 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s25 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v5.h, v36.h, s24 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, s25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v21.l, v5.h, s26 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0, v5.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v21.h, s24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v5, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v36.h, v5.h, s25 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s26 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v4.h, v36.h, s25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v5.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, s26 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v20.l, v4.h, s27 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0, v4.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v20.h, s25 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v4, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v36.h, v4.h, s26 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s27 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v3.h, v36.h, s26 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.h, v3.l, v19.l, s27 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v81, v81 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v36 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v51, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v28 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v52, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v29.l, v13.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v13.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v27 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v30.l, v13.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v51, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v29.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v28.l, v12.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v29.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v31.l, v12.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v26 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v51, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v28.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v26.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v52 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v51, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v25.l, v9.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v31.l, v10.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v12.l, v10.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.l, v10.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v27, v26 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v12.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v10.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v27 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v11.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v26 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v25, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v22.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v7.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v9.l, v7.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v10.l, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v11 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v7.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v8.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v12, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v6.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v7.l, v4.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v18.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v17 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v12, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v17, v16 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v2.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v8.l, v1.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v6.l, v0.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v9.l, v2.h, s3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v29 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v49 :: v_dual_mov_b32 v2, v48 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v39 :: v_dual_mov_b32 v4, v38 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v36 :: v_dual_mov_b32 v6, v35 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v34 :: v_dual_mov_b32 v8, v33 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v32 :: v_dual_mov_b32 v10, v31 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v30 :: v_dual_mov_b32 v12, v37 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v19.l, v83.h, s28 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v3, v3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0, v83.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v83, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.h, v2.l, v18.l, s28 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v1.l, v17.l, s40 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v32.h, v37.h, s44 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v36.h, v83.h, s27 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s40, 0, v82.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v85, v85 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v18.h, s26 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v32.l, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v83.h, s41 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v36.h, s28 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v31.h, v35.h, s42 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0, v39.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0, v19.h +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v18.l, v82.h, s29 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0, v38.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v31.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v34.h, v39.h, s42 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v48.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v82, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v33.h, v38.h, s29 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v34.l, v1.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v37.l, v48.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v36.h, v82.h, s44 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v33.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v49.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v50.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v35.l, v1.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v82.h, s40 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v51.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v54.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v49.l, v50.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s45 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v39.l, v49.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v52.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v51.l, v51.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v53.h +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v17.l, v19.h, s27 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v38.l, v0.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v69.l, v52.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v55.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v48.l, v1.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v36 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v69.h, v53.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v70.h, v55.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v66.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v36.h, v19.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v65.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v50.l, v2.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v70.l, v54.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v52.l, v0.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v16.h, v19.h, s28 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v53.l, v1.h, s8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v71.h, v65.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v54.l, v2.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v67.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v64.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v65.l, v1.h, s12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v67.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v55.l, v3.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v16.l, v96.h, s41 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v71.l, v64.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v80.l, v66.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v2.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v17.l, s4 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v96, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v64.l, v0.h, s11 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v66.l, v16.l, s13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v67.l, v15.l, s14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v68.l, v30.h, s15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v36.h, v96.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v16.l, v96.h, s43 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v17.h, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v32bf16: @@ -12956,753 +12767,697 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: scratch_load_b32 v68, off, s32 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v14 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v29 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v35.l, v36.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v37.l, v36.l ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v13 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v38.l, v36.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v12 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v28 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v27 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v55.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v86, v118 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v85.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v64.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v65.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v66.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v67.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v68.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v69.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v133.l, v70.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v134.l, v71.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v135.l, v80.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v144.l, v81.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v145.l, v82.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v146.l, v83.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v147.l, v84.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v119 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v128 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v130 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v131 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v132 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v133 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v134 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v135 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v144 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v145 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s63, v116, v86 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v86.l, v55.l, v32.l, s40 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v55.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v146 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v147 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v87, v118 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v96, v119 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s45, v98, v129 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s56, v101, v132 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s59, v112, v135 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s60, v113, v144 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v86.l, v32.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v86.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v34.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v36.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v39.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v50.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v51.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s46, v99, v130 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s61, v114, v145 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s62, v115, v146 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.l, v65.l, v34.l, s43 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v98.l, v67.l, v36.l, s45 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v101.l, v70.l, v39.l, s56 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v112.l, v81.l, v50.l, s59 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v113.l, v82.l, v51.l, s60 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.h, v55.l, s16 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v118 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v33.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v37.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v52.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v53.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0, v70.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0, v81.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v64.l, v33.l, s42 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v99.l, v68.l, v37.l, s46 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v114.l, v83.l, v52.l, s61 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v115.l, v84.l, v53.l, s62 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v96.l, v34.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v98.l, v36.l, s5 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v101.l, v39.l, s8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v39.l, v101.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v112.l, v50.l, s11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v112.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v113.l, v51.l, s12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v113.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v55 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v54.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v97, v128 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v116.l, v85.l, v54.l, s63 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v87.l, v33.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v99.l, v37.l, s6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v114.l, v52.l, s13 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v114.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v115.l, v53.l, s14 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v115.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v70.l, s23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v39 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v35.h, v81.l, s26 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v35.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s57, v102, v133 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v97.l, v66.l, v35.l, s44 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v116.l, v54.l, s15 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v54.l, v116.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v51 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v48.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v65.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v66.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s47, v100, v131 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s58, v103, v134 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v71.l, v48.l, s57 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v96.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v97.l, v35.l, s4 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v97.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v52 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v38.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v49.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v64.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0, v71.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v100.l, v69.l, v38.l, s47 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v103.l, v80.l, v49.l, s58 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v87.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v102.l, v48.l, s9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v102.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v30.h, v65.l, s18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v128 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v66.l, s19 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v129 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0, v80.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0, v85.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v38.l, s7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v103.l, v49.l, s10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v103.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v14.h, v64.l, s17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v119 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v34.h, v71.l, s24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v48 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v65 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v66 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v86.l, v13.h, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v80.l, s25 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v49 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v37.h, v85.l, s41 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v64 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v71 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v96.l, v30.h, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v97.l, v32.l, s4 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0, v67.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0, v68.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v98.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v99.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v87.l, v38.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v102.l, v38.h, s9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v67.l, s20 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v130 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v68.l, s21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v131 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0, v84.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v70 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v67 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0, v82.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v68 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v84.l, s29 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v101.l, v34.l, s8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0, v83.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v80 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v82.l, s27 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v115.l, v37.l, s14 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0, v69.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v36.h, v83.l, s28 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v103.l, v35.l, s10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v113.l, v36.l, s12 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v100.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.h, v69.l, s22 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v81 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.h, v114.l, v48.l, s13 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v132 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v69 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v33.h, s7 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v39, v39 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v39.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v9 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v8 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v24 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v23 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v22 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v21 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v20 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v7 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v19 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v18 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v17 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v15 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v30 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v17 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v16 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v83, v83 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v83.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v86, v86 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v36.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.h, v0.l, v16.l, s42 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s43, 0, v96.h ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v31 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v68 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v50.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v15.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v15.h, v68.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v31 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v98.l, v32.h, s5 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v51.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v99.l, v33.l, s6 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v54, v54 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v50.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v52, v53 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v51.l, v50.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v52 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v51.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v32.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v50.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v38, v53 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v112.l, v39.l, s11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v15.h, v51.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v31.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v14.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v29 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v50 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v30.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v33.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v15.h, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v117, v117 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v31.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v68.h, v36.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v36, v35 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0, v35.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v35.h, v36.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v51, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v28 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v14.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v31.l, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v31.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v14.h, v30.h, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v30.h, v36.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v36, v37 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s44, 0, v37.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v37.h, v36.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.l, v36.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v32.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v30.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v52, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v13.h, v29.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v29.h, v36.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v36, v38 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v12.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v29.l, v13.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v28.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v11 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v13.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v27 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v38.h, v36.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.l, v36.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v33.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v30.l, v13.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v51, v50 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v29.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v12.h, v28.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v28.h, v36.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v36, v39 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v28.l, v12.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v12.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v29.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v39.h, v36.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v34.l, v36.h, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v34.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v48, v48 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v31.l, v12.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v27.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v26 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v28.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v31.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v11.h, v27.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.h, v27.h, v36.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v36, v48 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v51, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v28.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v48.h, v36.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v35.l, v36.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v35.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v26.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v10.h, v26.h, s4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v26.h, v36.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s4, v36, v49 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v27.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v52 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v51, v50 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v49.h, v36.h, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v38.l, v36.h, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v38.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v25.l, v9.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v31.l, v10.h, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v28 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v12.l, v10.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v9.h, v25.h, s5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.h, v25.h, v36.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v36, v50 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v27, v26 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v50.h, v36.h, s5 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v48.l, v36.h, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v48.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v9.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v12.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v8.h, v24.h, s6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.h, v24.h, v36.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s6, v36, v51 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v10.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v51.h, v36.h, s6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v50.l, v36.h, s7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v50.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v11.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v27 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v7.h, v23.h, s7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v23.h, v36.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s7, v36, v52 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v11.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v11.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.h, v36.h, s7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v52.l, v36.h, s8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v52.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v12.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v26 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v25, v24 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v6.h, v22.h, s8 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.h, v22.h, v36.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s8, v36, v53 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v22.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v7.h, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v9.l, v7.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v10.l, v7.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.h, v36.h, s8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.h, v53.l, v36.h, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v53.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v54.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v5.h, v21.h, s9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.h, v36.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s9, v36, v54 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v7.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v20 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.h, v36.h, s9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v54.l, v36.h, s10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v54.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v55.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v4.h, v20.h, s10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.h, v20.h, v36.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s10, v36, v55 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v8.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.h, v36.h, s10 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.h, v55.l, v36.h, s11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v55.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v64.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v9.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v12, v11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v3.h, v19.h, s11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.h, v19.h, v36.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s11, v36, v64 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v3.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v6.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.h, v36.h, s11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v64.l, v36.h, s12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v64.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v65.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v7.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v2.h, v18.h, s12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.h, v18.h, v36.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s12, v36, v65 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v18.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.h, v36.h, s12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.h, v65.l, v36.h, s13 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v65.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v17 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v1.h, v17.h, s13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.h, v17.h, v36.h, s14 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v36.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s13, v36, v66 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.h, v36.h, s13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v66.l, v36.h, s14 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v66.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v67.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v0.h, v16.h, s14 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.h, v16.h, v36.h, s15 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v36.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v68 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s14, v36, v67 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.h, v15.l, v68.l, s15 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.h, v36.h, s14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v67.l, v36.h, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v67.l +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v82.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v68.l, v82.h, s15 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s15, v82, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v36.h, v82.h, s15 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v68.l, v82.h, s16 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v13 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v14.h, v36.h, s15 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v68.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v14.l, v30.l, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s15, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v30.l, v14.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v14.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v27 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s16, v14, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v82, v82 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v36.h, v14.h, s16 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v36.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v26 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v14.l, v14.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v12 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v25 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v13.h, v36.h, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v14.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.l, v29.l, s17 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v24 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s16, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v29.l, v13.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v13.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v23 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s17, v13, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v12, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v29.h, s16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v82, v82 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v36.h, v13.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v36.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v13.l, v13.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v11 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v21 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v12.h, v36.h, s17 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v13.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.l, v28.l, s18 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v20 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v28.l, v12.h, s19 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v19 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v12, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v28.h, s17 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v82, v82 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v36.h, v12.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v36.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v12.l, v12.h, s19 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v36.h, s18 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v12.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.l, v27.l, s19 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v27.l, v11.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0, v11.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v17, v16 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v27.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v11, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v36.h, v11.h, s19 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v11.l, v11.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.h, v10.h, v36.h, s19 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v11.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v26.l, s20 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v26.l, v10.h, s21 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0, v10.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v26.h, s19 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v10, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v36.h, v10.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v10.l, v10.h, s21 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v8 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.h, v9.h, v36.h, s20 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v25.l, s21 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v25.l, v9.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0, v9.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v25.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v9, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v36.h, v9.h, s21 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v9.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v7 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v36.h, s21 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v9.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v24.l, s22 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v24.l, v8.h, s23 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0, v8.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v24.h, s21 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v8, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v36.h, v8.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v8.l, v8.h, s23 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v7.h, v36.h, s22 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, s23 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v23.l, v7.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0, v7.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v23.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v7, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v36.h, v7.h, s23 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v6.h, v36.h, s23 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, s24 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v22.l, v6.h, s25 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0, v6.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v22.h, s23 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v6, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v36.h, v6.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s25 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v5.h, v36.h, s24 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v6.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, s25 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v21.l, v5.h, s26 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0, v5.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v2.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v8.l, v1.h, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v6.l, v0.h, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v9.l, v2.h, s3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v29 :: v_dual_mov_b32 v1, v49 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v48 :: v_dual_mov_b32 v3, v39 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v38 :: v_dual_mov_b32 v5, v36 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v35 :: v_dual_mov_b32 v7, v34 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v33 :: v_dual_mov_b32 v9, v32 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v31 :: v_dual_mov_b32 v11, v30 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v12, v37 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v21.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v5, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v36.h, v5.h, s25 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s26 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v4.h, v36.h, s25 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, s26 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v20.l, v4.h, s27 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0, v4.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v20.h, s25 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v4, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v36.h, v4.h, s26 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s27 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v3.h, v36.h, s26 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v4.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.h, v3.l, v19.l, s27 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v81, v81 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v19.l, v83.h, s28 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v3, v3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0, v83.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v83, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.h, v2.l, v18.l, s28 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v1.l, v17.l, s40 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v32.h, v37.h, s44 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v36.h, v83.h, s27 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s40, 0, v82.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v85, v85 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v18.h, s26 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v32.l, v1.l, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v83.h, s41 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v36.h, s28 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v3.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v31.h, v35.h, s42 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0, v39.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0, v19.h +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v18.l, v82.h, s29 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0, v38.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v31.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v34.h, v39.h, s42 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v48.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v82, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v33.h, v38.h, s29 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v34.l, v1.h, s2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v37.l, v48.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v36.h, v82.h, s44 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v33.l, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v49.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v50.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v35.l, v1.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v82.h, s40 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v51.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v54.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v49.l, v50.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s45 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v1.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v39.l, v49.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v52.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v51.l, v51.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v53.h +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v17.l, v19.h, s27 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v38.l, v0.h, s4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v69.l, v52.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v55.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v48.l, v1.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v36 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v69.h, v53.h, s2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v70.h, v55.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v66.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v36.h, v19.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v65.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v50.l, v2.h, s6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v70.l, v54.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v52.l, v0.h, s7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v16.h, v19.h, s28 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v53.l, v1.h, s8 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v71.h, v65.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v54.l, v2.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v67.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v64.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v65.l, v1.h, s12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v67.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v55.l, v3.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v16.l, v96.h, s41 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v71.l, v64.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v80.l, v66.h, s2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v17.l, s4 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v96, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v64.l, v0.h, s11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v66.l, v16.l, s13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v67.l, v15.l, s14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v68.l, v30.h, s15 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v36.h, v96.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v16.l, v96.h, s43 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v17.h, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v32bf16: @@ -14612,34 +14367,29 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 { ; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_bf16_no_ieee: @@ -14675,40 +14425,34 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_bf16_no_ieee: @@ -14949,58 +14693,46 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> % ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v3, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16_no_ieee: @@ -15057,62 +14789,56 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> % ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v3, v4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16_no_ieee: @@ -15458,66 +15184,67 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> % ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v7, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v9, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee: @@ -15589,77 +15316,80 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> % ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v7, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v9, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee: @@ -16117,85 +15847,83 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> % ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v10, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v9, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v8, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee: @@ -16290,99 +16018,98 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> % ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v10, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v9, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v7.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v8.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v8, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v5.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee: diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll index 678d0a4..416a601 100644 --- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll @@ -120,34 +120,29 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) { ; GFX11-TRUE16-LABEL: v_minimumnum_bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_bf16: @@ -183,40 +178,34 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_bf16: @@ -344,21 +333,21 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) { ; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_bf16_nnan: @@ -386,25 +375,25 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_bf16_nnan: @@ -639,58 +628,46 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v3, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16: @@ -747,62 +724,56 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v3, v4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16: @@ -1024,34 +995,29 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y) ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h ; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, s2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16_nnan: @@ -1097,36 +1063,35 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y) ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h ; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v3.h, v0.l, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.h, v0.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, s1 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16_nnan: @@ -1459,66 +1424,67 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v7, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v9, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16: @@ -1590,77 +1556,80 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v7, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v9, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16: @@ -1957,41 +1926,40 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y) ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v7, v6 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v9, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v0.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v0.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v2.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_nnan: @@ -2047,48 +2015,50 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y) ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v7, v6 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v9, v8 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v0.h, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v0.l, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_nnan: @@ -2528,85 +2498,83 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v10, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v9, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v8, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16: @@ -2701,99 +2669,98 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v10, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v9, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v7.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v8.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v8, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v5.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16: @@ -3181,53 +3148,52 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y) ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v7, v6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v6 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v1.h, s4 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v6, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v3.h -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.h, v0.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v0.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_nnan: @@ -3296,62 +3262,63 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y) ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l ; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v7, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v1.h, s4 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v6, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v1.l, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v6 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v3.h -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.h, v0.l, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.h, v0.l, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.h, v0.h, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_nnan: @@ -3984,125 +3951,120 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) { ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v9.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v2.h, v5.h, vcc_lo ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v13, v13 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v8 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v9.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v4.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v10.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v9.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v7.l ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v8.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v3.h, s2 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v13, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v7.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v9.l, v8.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v13.l, v8.l, s3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v16 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v12.l, v10.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v10.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v7 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v12.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v4.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 ; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v3.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v0.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v11.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v9.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v8.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.l, v5.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v5.l, v12.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v12.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v5, v5 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v9.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v12, v12 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v4.l, v1.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v11.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v4.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v1, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.h, v1.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v5, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.h, v10.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v9.h, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v1.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v9.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v1.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.l, v0.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v0.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v0, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v9.h, v0.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v8.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v9.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v8.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v6.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s4 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v6bf16: @@ -4233,142 +4195,141 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v9.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v2.h, v5.h, vcc_lo ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v13, v13 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v8 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v8.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v3.h, s2 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v13, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v9.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v7.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v9.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v3.h, v10.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v4.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v9.l, v8.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v13.l, v8.l, s3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v7.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v15 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v4.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v10.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v12.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v9.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v7 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v7.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v0.h, v3.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v12.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v11.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v9.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v4.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v8.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.l, v5.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v5.l, v12.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v12.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v12.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v5, v5 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v9.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v12, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v4.l, v1.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v11.h, s7 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v1, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.h, v1.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v9.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v5, v5 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.h, v10.h, s6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v9.h, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v9.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v1.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.l, v0.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v0, v9 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v5.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v9.h, v0.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v8.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v9.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v8.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v6.l, v0.h, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v13.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v3.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s4 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v6bf16: @@ -5250,171 +5211,160 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) { ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v5 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v3.h, v7.h, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v15, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v5.h, v12.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v13, v18 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v10.l, v8.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v11.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v9.l, s3 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v13.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v15.l, v9.l, s4 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v17, v17 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v17, v17 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v8.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v6.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v13.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v13.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v9.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v1.h, v5.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v5.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v14.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v0.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v4.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v15.h, v12.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v11.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.l, v7.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v7.l, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v12.h, v16.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v7, v7 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.h, v12.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v16, v16 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v6.l, v2.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v2.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v10.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v6.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v2, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v12.h, v2.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v12.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v7, v7 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v13.h, s8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.h, v12.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, s2 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.h, v11.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v15.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.h, v10.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v14.l, v12.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v8.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v8.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v7.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v15, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.l, v2.l, s3 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v6.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v3.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.l, v4.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v1.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v5.l, v1.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v1.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v1, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v12.h, v1.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v12.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v7, v7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v12.h, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v12.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v8 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.l, v0.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s6, v0, v12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v12.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v12.h, v0.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v11.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.h, v11.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v14.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v15.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v12.h, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.h, v14.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v15.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v1.h, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v0.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v5.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v8bf16: @@ -5577,201 +5527,187 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v5 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v12.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v3.h, v7.h, vcc_lo +; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v15, v17 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v5.h, v12.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v13, v18 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v10.l, v8.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v11.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v9.l, s3 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v13.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v15.l, v9.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.h, v11.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v17, v17 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v17, v17 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v13.l, v8.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v15.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.h, v10.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v14.l, v12.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v9.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v8.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v12.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v2.h, v6.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v6.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v13.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v14.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v8.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v8.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v8.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v7.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v13.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v9.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v12.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v1.h, v5.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v5.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v14 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v14.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v10.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v0.h, v4.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v15, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v4.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v15 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v15.h, v12.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.l, v2.l, s3 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v11.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.h, v7.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.l, v7.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v7.l, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v12.h, v16.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v7, v7 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.h, v12.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v16, v16 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v6.l, v2.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v2.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v2, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v12.h, v2.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v7, v7 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v13.h, s8 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.h, v12.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.l, v2.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v5.l, v1.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v1.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v6.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v1, v12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v12.h, v1.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v12.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v1.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v7, v7 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v12.h, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v12.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.l, v0.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v5.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s6, v0, v12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v12.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v3.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v12.h, v0.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v11.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.l, v4.h, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v1.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.h, v11.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v14.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v15.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v12.h, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v0.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.h, v14.h, s6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v15.h, s7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v1.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v0.h, s3 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v9 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v5.l, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v8bf16: @@ -7391,341 +7327,314 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) { ; GFX11-TRUE16-LABEL: v_minimumnum_v16bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v18, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v17 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v13 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v19 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v18 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v13.h, v20.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v24, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v19.l, v7.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v19.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v23.l, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v25 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v20.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v19.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v6.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v11 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v3.h, v11.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v26, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v19.l, v22.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v11.h, v20.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v7.l, v5.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v5.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v7.h, v15.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v16.l +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v15.h, v16.h, vcc_lo +; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v17 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l ; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v19.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v16.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.h, v16.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v21.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v20.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v7.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v22.l, v20.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v1.h, v9.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v21.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v9.h, v23.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v20.l, v21.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v27, v25 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v7.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v7.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v5.l, v23.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v23.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v21.l, v23.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v20.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.l, v16.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v7.l, v19.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v19.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v14.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v19.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v24, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v6.l, v16.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.l, v16.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v14.l, v17.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v16.l, v17.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v14.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v14.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v17.l, v16.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v6.h, v14.h, s0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v8 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v29, v29 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v14.h, v16.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v32, v32 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v18 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.h, v16.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v18.l, v16.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v18.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v5.h, v13.h, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.h, v13.h, v16.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v16.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v32, v32 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v25 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v25.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v32, v32 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v25.h, v16.h, s1 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v16.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v20.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v32, v32 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v16.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v4.h, v12.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.h, v12.h, v16.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v26.h, v16.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.l, v16.h, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v21.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.h, v11.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v16, v27 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v27.h, v16.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.l, v16.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v22.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v2.h, v10.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v10.h, v16.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v16, v28 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v28.h, v16.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.l, v16.h, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v23.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v1.h, v9.h, s5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v9.h, v16.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v16, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v29.h, v16.h, s5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.l, v16.h, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v24.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v30, v30 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v0.h, v8.h, s6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v8.h, v16.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s6, v16, v30 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v30.h, v16.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v15.h, v16.h, s7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v15.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v15.l, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v16.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v7.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v7.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s7, v7, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v16.h, v7.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v6.h, v16.h, s7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v7.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, s8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v14.l, v6.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v14.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s8, v6, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v6.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v5.h, v16.h, s8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v6.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v13.l, v5.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v13.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s9, v5, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v16.h, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, s10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v12.l, v4.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v12.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s10, v4, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v16.h, v4.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v16.h, s10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, s11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v11.l, v3.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v11.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s11, v3, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.h, v3.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v3.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.h, v16.h, s11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, s12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v10.l, v2.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v2.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v10.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s12, v2, v16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v16.h, v2.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v31, v31 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v16.h, s12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, s13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v9.l, v1.h, s14 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v1.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v9.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s13, v1, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v0.l, v8.l, s14 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v1.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v17.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v1.h, s15 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.l, v17.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v18.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v20.h, v25.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v32.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.h, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.h, v18.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v20.l, v1.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v8.l, v32.h, s14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v18.l, v0.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.h, v26.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v28.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v32, v16 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v21.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v29.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v23.h, v28.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.h, v32.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v22.h, v27.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v29.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.l, v32.h, s15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v23.l, v1.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v22.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v8.h, s5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v25.l, v30.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v16.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v8.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.h, v0.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v12.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.l, v4.h, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v17, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v10.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v8.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v12.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v22, v17 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v11.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v2.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.l, v1.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v10.l, v0.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v13.l, v2.h, s3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v19 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v8.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v16bf16: @@ -8044,406 +7953,355 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v18, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v15 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v17 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v13 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v15.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v5.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v19 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.h, v14.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v7.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v18.h, v13.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v13.h, v20.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v24, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v19.l, v7.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v19.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v23.l, v7.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v25 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v4.h, v12.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v20.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v19.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v12.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v6.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v20 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v3.h, v11.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v23.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v26, v21 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v19.l, v22.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v11.h, v20.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v7.l, v5.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v5.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v13 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v7.h, v15.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v15.h, v16.h, vcc_lo +; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v16.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v3 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v17 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l ; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v19.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v23 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v16.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v10.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v25 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.h, v16.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v21.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v20.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v7.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v22.l, v20.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v22.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v1.h, v9.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v21.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v9.h, v23.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v20.l, v21.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v27, v25 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v22.l, v19.l, vcc_lo -; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v7.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v20.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v0.h, v8.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v7.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v25 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v5.l, v23.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v23.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v19.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v21.l, v23.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v19.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v15 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v20.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v15.l, v16.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v7.l, v19.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v19.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v14.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v19.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v24, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v7.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v6.l, v16.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v13 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.l, v16.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v13.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v14.l, v17.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v13.l, v15.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v16.l, v17.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v14.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v15.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v17.l, v16.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v6.h, v14.h, s0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v8 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v29, v29 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v16.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v14.h, v16.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v7 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v32, v32 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v18 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v14 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v14.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.h, v16.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v18.l, v16.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v18.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v15.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v5.h, v13.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.h, v13.h, v16.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v32, v32 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v25 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v25.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v32, v32 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v25.h, v16.h, s1 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v16.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v20.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v32, v32 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v32.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.l, v15.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v4.h, v12.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.h, v12.h, v16.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v26 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v12.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.l, v4.h, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v17, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v26.h, v16.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.l, v16.h, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v21.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v10.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v3.h, v11.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v16, v27 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v12.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v27.h, v16.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.l, v16.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v22.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v23, v23 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v10.l, v3.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v8.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v12.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v22, v17 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v2.h, v10.h, s4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v10.h, v16.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v16, v28 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v10.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v28.h, v16.h, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.l, v16.h, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v23.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v24, v24 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v16 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v1.h, v9.h, s5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v9.h, v16.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v16, v29 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v29.h, v16.h, s5 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.l, v16.h, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v24.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v30, v30 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v0.h, v8.h, s6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v8.h, v16.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s6, v16, v30 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v30.h, v16.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v15.h, v16.h, s7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v15.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v15.l, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v15.l, v7.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v7.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s7, v7, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v16.h, v7.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v16.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v6.h, v16.h, s7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, s8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v14.l, v6.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v6.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v11.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v14.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s8, v6, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v6.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v5.h, v16.h, s8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v6.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v13.l, v5.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v5.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v13.h, s8 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s9, v5, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.h, v5.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v16.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v16.h, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, s10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v12.l, v4.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v4.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v12.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s10, v4, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v16.h, v4.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v16.h, s10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v4.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, s11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v11.l, v3.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v3.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v11.h, s10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s11, v3, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.h, v3.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v16.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.l, v3.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.h, v16.h, s11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v3.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, s12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v10.l, v2.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v2.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v2.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.l, v1.h, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v10.l, v0.h, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v13.l, v2.h, s3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v14 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v10.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s12, v2, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v16.h, v2.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v2.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v31, v31 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v1.h, v16.h, s12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, s13 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v16 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v19 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v9.l, v1.h, s14 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v1.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v8 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v9.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s13, v1, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v0.l, v8.l, s14 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v16.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v31, v31 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v1.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v17.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v1.h, s15 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.l, v17.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v18.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v20.h, v25.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v32.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.h, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v1.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v19.h, v18.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v26.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v20.l, v1.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v16 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v8.l, v32.h, s14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v18.l, v0.h, s0 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.h, v26.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v28.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v32, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v21.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v29.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v23.h, v28.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.h, v32.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v22.h, v27.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v29.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v8.l, v32.h, s15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v23.l, v1.h, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v22.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v8.h, s5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v25.l, v30.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v16.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v8.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.h, v0.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v16 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v8.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v16bf16: @@ -11736,666 +11594,619 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) { ; GFX11-TRUE16-LABEL: v_minimumnum_v32bf16: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: scratch_load_b32 v68, off, s32 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v14 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v29 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v36.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v36.l ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v13 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v36.l +; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v12 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v28 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v27 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v55.l -; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v86, v118 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v85.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v64.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v65.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v66.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v67.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v68.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v69.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v70.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v71.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v80.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v81.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v82.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v83.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v84.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v119 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v128 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v130 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v131 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v132 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v133 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v134 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v135 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v144 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v145 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s63, v116, v86 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v86.l, v55.l, v32.l, s40 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v13 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v55.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v146 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v147 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v87, v118 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v96, v119 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s45, v98, v129 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s56, v101, v132 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s59, v112, v135 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s60, v113, v144 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v86.l, v32.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v86.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v34.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v36.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v39.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v50.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v51.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s46, v99, v130 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s61, v114, v145 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s62, v115, v146 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.l, v65.l, v34.l, s43 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v98.l, v67.l, v36.l, s45 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v101.l, v70.l, v39.l, s56 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v112.l, v81.l, v50.l, s59 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v113.l, v82.l, v51.l, s60 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.h, v55.l, s16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v118 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v33.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v37.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v52.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v53.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0x8000, v70.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0x8000, v81.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v64.l, v33.l, s42 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v99.l, v68.l, v37.l, s46 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v114.l, v83.l, v52.l, s61 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v115.l, v84.l, v53.l, s62 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v96.l, v34.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v98.l, v36.l, s5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v101.l, v39.l, s8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v101.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v112.l, v50.l, s11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v112.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v113.l, v51.l, s12 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v113.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v55 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v54.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v97, v128 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v116.l, v85.l, v54.l, s63 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v87.l, v33.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v99.l, v37.l, s6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v114.l, v52.l, s13 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v114.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v115.l, v53.l, s14 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v115.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v70.l, s23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v39 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v35.h, v81.l, s26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v35.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s57, v102, v133 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v97.l, v66.l, v35.l, s44 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v116.l, v54.l, s15 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v116.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v51 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v48.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v65.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v66.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s47, v100, v131 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s58, v103, v134 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v71.l, v48.l, s57 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v96.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v97.l, v35.l, s4 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v97.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v52 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v38.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v49.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v64.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0x8000, v71.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v100.l, v69.l, v38.l, s47 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v103.l, v80.l, v49.l, s58 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v87.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v102.l, v48.l, s9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v102.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v30.h, v65.l, s18 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v128 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v66.l, s19 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v129 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0x8000, v80.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0x8000, v85.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v38.l, s7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v103.l, v49.l, s10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v103.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v14.h, v64.l, s17 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v119 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v34.h, v71.l, s24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v48 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v65 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v66 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v86.l, v13.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v80.l, s25 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v49 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v37.h, v85.l, s41 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v64 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v71 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v96.l, v30.h, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v97.l, v32.l, s4 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0x8000, v67.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0x8000, v68.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v98.l -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v99.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v87.l, v38.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v102.l, v38.h, s9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v67.l, s20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v130 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v68.l, s21 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v131 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0x8000, v84.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v70 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v67 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0x8000, v82.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v68 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v84.l, s29 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v101.l, v34.l, s8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0x8000, v83.l -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v80 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v82.l, s27 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v115.l, v37.l, s14 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0x8000, v69.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v36.h, v83.l, s28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v103.l, v35.l, s10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v113.l, v36.l, s12 -; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v100.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.h, v69.l, s22 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v81 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.h, v114.l, v48.l, s13 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v132 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v69 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v33.h, s7 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v39, v39 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v9 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v8 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v24 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v23 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v22 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v21 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v7 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v6 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v5 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v4 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v3 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v19 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v18 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v17 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v15 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v30 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v2 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v17 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v16 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v83, v83 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v86, v86 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.h, v0.l, v16.l, s42 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s43, 0x8000, v96.h ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) -; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v31 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v68 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v15.h, v68.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v68.h, v36.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v36, v35 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0x8000, v35.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v35.h, v36.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v31.l, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v31.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v14.h, v30.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v30.h, v36.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v36, v37 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s44, 0x8000, v37.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v37.h, v36.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.l, v36.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v32.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v13.h, v29.h, s1 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v29.h, v36.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v36, v38 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v38.h, v36.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.l, v36.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v33.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v12.h, v28.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v28.h, v36.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v36, v39 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v39.h, v36.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v34.l, v36.h, s3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v34.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v48, v48 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v36.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v50.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v15.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v31 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v98.l, v32.h, s5 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v51.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v99.l, v33.l, s6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v54, v54 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v50.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v52, v53 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v11.h, v27.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.h, v27.h, v36.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v36, v48 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v48.h, v36.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v35.l, v36.h, s4 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v35.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v36.l ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v51.l, v50.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v52 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v51.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v32.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v50.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v10.h, v26.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v26.h, v36.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v36, v49 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v49.h, v36.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v38.l, v36.h, s5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v38.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v9.h, v25.h, s5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.h, v25.h, v36.h, s6 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v36, v50 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v50.h, v36.h, s5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v48.l, v36.h, s6 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v48.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v8.h, v24.h, s6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.h, v24.h, v36.h, s7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s6, v36, v51 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v51.h, v36.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v50.l, v36.h, s7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v50.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v7.h, v23.h, s7 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v23.h, v36.h, s8 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s7, v36, v52 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.h, v36.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v52.l, v36.h, s8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v52.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v6.h, v22.h, s8 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.h, v22.h, v36.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s8, v36, v53 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.h, v36.h, s8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.h, v53.l, v36.h, s9 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v53.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v5.h, v21.h, s9 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.h, v36.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s9, v36, v54 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.h, v36.h, s9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v54.l, v36.h, s10 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v54.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v4.h, v20.h, s10 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.h, v20.h, v36.h, s11 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s10, v36, v55 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.h, v36.h, s10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.h, v55.l, v36.h, s11 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v55.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v3.h, v19.h, s11 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.h, v19.h, v36.h, s12 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s11, v36, v64 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.h, v36.h, s11 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v64.l, v36.h, s12 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v64.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v2.h, v18.h, s12 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.h, v18.h, v36.h, s13 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s12, v36, v65 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.h, v36.h, s12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.h, v65.l, v36.h, s13 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v65.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v1.h, v17.h, s13 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.h, v17.h, v36.h, s14 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s13, v36, v66 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.h, v36.h, s13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v66.l, v36.h, s14 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v66.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v0.h, v16.h, s14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.h, v16.h, v36.h, s15 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v68 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s14, v36, v67 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.h, v15.l, v68.l, s15 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v14 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.h, v36.h, s14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v67.l, v36.h, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v67.l +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v82.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v68.l, v82.h, s15 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s15, v82, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v36.h, v82.h, s15 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v68.l, v82.h, s16 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v13 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v14.h, v36.h, s15 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v68.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v14.l, v30.l, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s15, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v30.l, v14.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v14.h ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v38, v53 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v112.l, v39.l, s11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v15.h, v51.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v31.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v14.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v29 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v50 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v30.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v33.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v15.h, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v117, v117 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v31.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v27 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s16, v14, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v82, v82 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v36.h, v14.h, s16 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v26 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v14.l, v14.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v12 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v25 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v13.h, v36.h, s16 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v14.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.l, v29.l, s17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v24 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s16, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v29.l, v13.h, s18 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v13.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v23 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s17, v13, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v29.h, s16 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v82, v82 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v36.h, v13.h, s17 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v13.l, v13.h, s18 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v11 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v21 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v12.h, v36.h, s17 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v13.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.l, v28.l, s18 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v20 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v28.l, v12.h, s19 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v12.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v82, v82 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v19 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v12, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v28.h, s17 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v82, v82 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v36.h, v12.h, s18 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v12.l, v12.h, s19 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v36.h, s18 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v12.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.l, v27.l, s19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v27.l, v11.h, s20 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0x8000, v11.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v27.h, s18 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v11, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v36.h, v11.h, s19 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v11.l, v11.h, s20 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v9 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.h, v10.h, v36.h, s19 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v11.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v26.l, s20 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v26.l, v10.h, s21 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0x8000, v10.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v26.h, s19 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v10, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v36.h, v10.h, s20 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v10.l, v10.h, s21 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.h, v9.h, v36.h, s20 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v25.l, s21 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v25.l, v9.h, s22 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0x8000, v9.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v25.h, s20 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v9, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v36.h, v9.h, s21 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v9.h, s22 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v36.h, s21 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v9.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v24.l, s22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v24.l, v8.h, s23 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0x8000, v8.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v24.h, s21 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v8, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v36.h, v8.h, s22 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v8.l, v8.h, s23 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v7.h, v36.h, s22 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v8.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, s23 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v23.l, v7.h, s24 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0x8000, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v23.h, s22 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v7, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v36.h, v7.h, s23 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s24 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v6.h, v36.h, s23 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v7.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, s24 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v22.l, v6.h, s25 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v22.h, s23 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v6, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v36.h, v6.h, s24 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s25 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v5.h, v36.h, s24 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, s25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v21.l, v5.h, s26 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0x8000, v5.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v21.h, s24 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v5, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v36.h, v5.h, s25 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s26 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v4.h, v36.h, s25 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v5.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, s26 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v36 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v20.l, v4.h, s27 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0x8000, v4.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v20.h, s25 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v4, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v36.h, v4.h, s26 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s27 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v81, v81 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v3.h, v36.h, s26 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v4.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.h, v3.l, v19.l, s27 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v81, v81 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v36 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v51, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v28 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v52, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v29.l, v13.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v28.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v13.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v27 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v30.l, v13.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v51, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v29.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v28.l, v12.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v29.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v31.l, v12.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v27.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v26 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v28.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v31.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v51, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v28.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v26.l, v10.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v52 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v51, v50 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v25.l, v9.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v31.l, v10.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v12.l, v10.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.l, v10.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v24 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v27, v26 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v12.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v10.l, v9.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v23 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v10.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v27 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v11.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v11.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v26 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v25, v24 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v22.l, v6.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v7.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v9.l, v7.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v10.l, v7.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v21 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v11 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v22 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v7.l, v6.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v20 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v21 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v8.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v12, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v18 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v6.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v7.l, v4.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v18.l, v2.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v17 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v9.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.l, v0.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v12, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v17, v16 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v2.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v8.l, v1.h, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v6.l, v0.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v9.l, v2.h, s3 -; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v29 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v49 :: v_dual_mov_b32 v2, v48 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v39 :: v_dual_mov_b32 v4, v38 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v36 :: v_dual_mov_b32 v6, v35 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v34 :: v_dual_mov_b32 v8, v33 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v32 :: v_dual_mov_b32 v10, v31 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v30 :: v_dual_mov_b32 v12, v37 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v19.l, v83.h, s28 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v3, v3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0x8000, v83.h +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v36.l +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v83, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.h, v2.l, v18.l, s28 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v1.l, v17.l, s40 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v32.h, v37.h, s44 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v36.h, v83.h, s27 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s40, 0x8000, v82.h +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v85, v85 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v18.h, s26 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v32.l, v1.l, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v83.h, s41 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v36.h, s28 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v3.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v31.h, v35.h, s42 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0x8000, v39.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0x8000, v19.h +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v18.l, v82.h, s29 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0x8000, v38.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v31.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v34.h, v39.h, s42 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v82, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v33.h, v38.h, s29 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v34.l, v1.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v37.l, v48.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v36.h, v82.h, s44 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v33.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v49.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v50.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v35.l, v1.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v82.h, s40 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v51.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v54.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v49.l, v50.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s45 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v39.l, v49.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v52.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v51.l, v51.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v53.h +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v17.l, v19.h, s27 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v38.l, v0.h, s4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v69.l, v52.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v55.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v48.l, v1.h, s5 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v36 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v69.h, v53.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v70.h, v55.h, s4 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v66.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v36.h, v19.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v65.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v50.l, v2.h, s6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v70.l, v54.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v52.l, v0.h, s7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v16.h, v19.h, s28 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v53.l, v1.h, s8 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v71.h, v65.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v54.l, v2.h, s9 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v67.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.h +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v64.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v65.l, v1.h, s12 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v67.h, s3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v55.l, v3.h, s10 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v16.l, v96.h, s41 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v71.l, v64.h, s0 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v80.l, v66.h, s2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v2.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v17.l, s4 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v96, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v64.l, v0.h, s11 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v66.l, v16.l, s13 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v67.l, v15.l, s14 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v68.l, v30.h, s15 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v36.h, v96.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v16.l, v96.h, s43 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v36.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v17.h, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v32bf16: @@ -13011,753 +12822,697 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: scratch_load_b32 v68, off, s32 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v14 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v29 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v35.l, v36.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v37.l, v36.l ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v13 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v38.l, v36.l +; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v12 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v28 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v27 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v55.l -; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v86, v118 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v85.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v64.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v65.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v66.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v67.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v68.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v69.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v133.l, v70.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v134.l, v71.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v135.l, v80.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v144.l, v81.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v145.l, v82.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v146.l, v83.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v147.l, v84.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v119 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v128 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v130 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v131 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v132 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v133 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v134 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v135 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v144 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v145 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s63, v116, v86 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v86.l, v55.l, v32.l, s40 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v13 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v55.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v146 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v147 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v87, v118 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v96, v119 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s45, v98, v129 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s56, v101, v132 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s59, v112, v135 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s60, v113, v144 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v86.l, v32.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v86.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v34.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v36.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v39.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v50.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v51.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s46, v99, v130 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s61, v114, v145 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s62, v115, v146 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.l, v65.l, v34.l, s43 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v98.l, v67.l, v36.l, s45 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v101.l, v70.l, v39.l, s56 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v112.l, v81.l, v50.l, s59 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v113.l, v82.l, v51.l, s60 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.h, v55.l, s16 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v118 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v33.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v37.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v52.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v53.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0x8000, v70.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0x8000, v81.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v64.l, v33.l, s42 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v99.l, v68.l, v37.l, s46 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v114.l, v83.l, v52.l, s61 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v115.l, v84.l, v53.l, s62 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v96.l, v34.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v98.l, v36.l, s5 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v101.l, v39.l, s8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v39.l, v101.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v112.l, v50.l, s11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v112.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v113.l, v51.l, s12 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v113.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v55 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v54.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v97, v128 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v116.l, v85.l, v54.l, s63 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v87.l, v33.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v99.l, v37.l, s6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v114.l, v52.l, s13 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v114.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v115.l, v53.l, s14 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v115.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v70.l, s23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v39 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v35.h, v81.l, s26 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v35.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s57, v102, v133 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v97.l, v66.l, v35.l, s44 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v116.l, v54.l, s15 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v54.l, v116.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v51 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v48.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v65.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v66.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s47, v100, v131 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s58, v103, v134 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v71.l, v48.l, s57 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v96.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v97.l, v35.l, s4 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v97.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v52 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v38.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v49.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v64.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0x8000, v71.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v100.l, v69.l, v38.l, s47 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v103.l, v80.l, v49.l, s58 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v87.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v102.l, v48.l, s9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v102.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v30.h, v65.l, s18 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v128 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v66.l, s19 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v129 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0x8000, v80.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0x8000, v85.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v38.l, s7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v103.l, v49.l, s10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v103.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v14.h, v64.l, s17 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v119 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v34.h, v71.l, s24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v48 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v65 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v66 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v86.l, v13.h, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v80.l, s25 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v49 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v37.h, v85.l, s41 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v64 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v71 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v96.l, v30.h, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v97.l, v32.l, s4 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0x8000, v67.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0x8000, v68.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v98.l -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v99.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v87.l, v38.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v102.l, v38.h, s9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v67.l, s20 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v130 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v68.l, s21 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v131 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0x8000, v84.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v70 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v67 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0x8000, v82.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v68 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v84.l, s29 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v101.l, v34.l, s8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0x8000, v83.l -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v80 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v82.l, s27 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v115.l, v37.l, s14 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0x8000, v69.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v36.h, v83.l, s28 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v103.l, v35.l, s10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v113.l, v36.l, s12 -; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v100.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.h, v69.l, s22 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v81 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.h, v114.l, v48.l, s13 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v132 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v69 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v100.l, v33.h, s7 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v39, v39 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v39.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v9 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v8 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v24 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v23 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v22 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v21 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v20 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v7 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v6 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v5 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v4 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v3 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v19 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v18 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v17 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v15 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v30 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v2 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v17 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v16 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v83, v83 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v83.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v86, v86 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v36.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.h, v0.l, v16.l, s42 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s43, 0x8000, v96.h ; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v31 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v68 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v15.h, v31.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v50.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v15.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v15.h, v68.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v31.h, v50.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v54 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v31 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v98.l, v32.h, s5 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v51.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v99.l, v33.l, s6 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v54, v54 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v50.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v52, v53 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v15.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v51.l, v50.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v52 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v51.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v32.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v50.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v38, v53 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v112.l, v39.l, s11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v15.h, v51.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v31.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v14.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v29 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v50 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v30.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v33.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v15.h, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v117, v117 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v31.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v68.h, v36.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v36, v35 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0x8000, v35.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v116.l, v49.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v35.h, v36.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v51, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v30.l, v14.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v28 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v31.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v31.l, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v31.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v14.h, v30.h, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v30.h, v36.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v36, v37 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s44, 0x8000, v37.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v37.h, v36.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.l, v36.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v32.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v31.l, v14.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v30.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v52, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v13.h, v29.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v29.h, v36.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v36, v38 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v12.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v30.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v29.l, v13.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v28.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v11 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v15.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v13.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v31.l, v12.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v27 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v38.h, v36.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v33.l, v36.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v33.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v30.l, v13.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v51, v50 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v53 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v29.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v12.h, v28.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v28.h, v36.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v36, v39 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v28.l, v12.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v12.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v29.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v39.h, v36.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v34.l, v36.h, s3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v34.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v48, v48 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v31.l, v12.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v27.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v26 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v28.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v31.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v11.h, v27.h, s3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.h, v27.h, v36.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v36, v48 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v30.l, v11.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v51, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v28.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v48.h, v36.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v35.l, v36.h, s4 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v35.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v49, v49 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v26.l, v10.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v27.l, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v12.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v10.h, v26.h, s4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v26.h, v36.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s4, v36, v49 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v28, v28 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v26.l, v11.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v27.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v52 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v51, v50 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v49.h, v36.h, s4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v38.l, v36.h, s5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v38.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v50, v50 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v25.l, v9.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v31.l, v10.h, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v27.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v28 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v12.l, v10.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v9.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v26.l, v9.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v9.h, v25.h, s5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.h, v25.h, v36.h, s6 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v36, v50 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.l, v10.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v24 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v25.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v27, v26 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v50.h, v36.h, s5 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v48.l, v36.h, s6 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v48.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v51, v51 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v9.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v12.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v24.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v8.h, v24.h, s6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.h, v24.h, v36.h, s7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s6, v36, v51 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v10.l, v9.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v12.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v23 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v10.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v51.h, v36.h, s6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v50.l, v36.h, s7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v50.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v52, v52 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v11.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v27 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v25.l, v8.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v7.h, v23.h, s7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v23.h, v36.h, s8 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s7, v36, v52 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v11.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v11.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.h, v36.h, s7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v52.l, v36.h, s8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v52.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v53, v53 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s7, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v11.l, v8.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v12.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v26 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v25, v24 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v6.h, v22.h, s8 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.h, v22.h, v36.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s8, v36, v53 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v22.l, v6.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v7.h, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v12.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v9.l, v7.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v11.l, v6.h, vcc_lo -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v22 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v10.l, v7.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v21 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.h, v36.h, s8 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.h, v53.l, v36.h, s9 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v53.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v54, v54 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v54.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s8, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v9.l, s1 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v21.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v5.h, v21.h, s9 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.h, v36.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s9, v36, v54 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v7.l, v6.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v20 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.h, v36.h, s9 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v54.l, v36.h, s10 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v54.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v55, v55 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v55.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s9, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v21 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v10.l, v5.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v4.h, v20.h, s10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.h, v20.h, v36.h, s11 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s10, v36, v55 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v20.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v8.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v19 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.h, v36.h, s10 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.h, v55.l, v36.h, s11 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v55.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v64, v64 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v64.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s10, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v9.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v12, v11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v3.h, v19.h, s11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.h, v19.h, v36.h, s12 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s11, v36, v64 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v19.l, v3.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v9.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v18 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v6.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.h, v36.h, s11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v64.l, v36.h, s12 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v64.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v65, v65 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v65.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s11, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v7.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v2.h, v18.h, s12 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.h, v18.h, v36.h, s13 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s12, v36, v65 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v18.l, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v8.l, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v3.h, v6.l, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.h, v36.h, s12 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.h, v65.l, v36.h, s13 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v65.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v66, v66 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s12, 0, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v17 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v9.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v1.h, v17.h, s13 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.h, v17.h, v36.h, s14 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s13, v36, v66 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.h, v36.h, s13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v66.l, v36.h, s14 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v66.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v67, v67 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v67.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s13, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v0.h, v16.h, s14 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.h, v16.h, v36.h, s15 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v68 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s14, v36, v67 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.h, v15.l, v68.l, s15 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.h, v36.h, s14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v67.l, v36.h, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v67.l +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v82.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s14, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v68.l, v82.h, s15 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s15, v82, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v36.h, v82.h, s15 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v68.l, v82.h, s16 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v13 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v14.h, v36.h, s15 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v68.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v14.l, v30.l, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s15, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v30.l, v14.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v14.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v27 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s16, v14, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v82, v82 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v36.h, v14.h, s16 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v26 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v14.l, v14.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v12 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v25 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v13.h, v36.h, s16 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v14.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v13.l, v29.l, s17 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v24 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s16, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v29.l, v13.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v13.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v23 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s17, v13, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v3.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v16.l, v0.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v12, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v29.h, s16 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v82, v82 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v36.h, v13.h, s17 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v22 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v13.l, v13.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v11 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v21 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v12.h, v36.h, s17 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v13.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.l, v28.l, s18 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v20 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v28.l, v12.h, s19 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v12.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v82, v82 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v19 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v12, v36 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v4.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v28.h, s17 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v82, v82 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v36.h, v12.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v12.l, v12.h, s19 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v11.h, v36.h, s18 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v12.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.l, v27.l, s19 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v27.l, v11.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0x8000, v11.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v2.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v17, v16 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v27.h, s18 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v11, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v36.h, v11.h, s19 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v11.l, v11.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v9 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.h, v10.h, v36.h, s19 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v11.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.l, v26.l, s20 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v26.l, v10.h, s21 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0x8000, v10.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v26.h, s19 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v10, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v36.h, v10.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 0x8000, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v10.l, v10.h, s21 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v8 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.h, v9.h, v36.h, s20 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.l, v25.l, s21 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v25.l, v9.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0x8000, v9.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v25.h, s20 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v9, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v36.h, v9.h, s21 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v9.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v7 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v36.h, s21 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v9.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.l, v24.l, s22 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v24.l, v8.h, s23 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0x8000, v8.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v24.h, s21 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v8, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v36.h, v8.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 0x8000, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v8.l, v8.h, s23 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v7.h, v36.h, s22 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, s23 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v23.l, v7.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0x8000, v7.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v23.h, s22 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v7, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v36.h, v7.h, s23 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v7.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v5 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v6.h, v36.h, s23 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, s24 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v22.l, v6.h, s25 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0x8000, v6.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v22.h, s23 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v6, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v36.h, v6.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 0x8000, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v6.h, s25 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v5.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v5.h, v36.h, s24 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v6.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, s25 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v36.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v21.l, v5.h, s26 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0x8000, v5.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v2.l, v1.l, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v8.l, v1.h, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v6.l, v0.h, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v9.l, v2.h, s3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v29 :: v_dual_mov_b32 v1, v49 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v48 :: v_dual_mov_b32 v3, v39 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v38 :: v_dual_mov_b32 v5, v36 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v35 :: v_dual_mov_b32 v7, v34 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v33 :: v_dual_mov_b32 v9, v32 -; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v31 :: v_dual_mov_b32 v11, v30 -; GFX12-TRUE16-NEXT: v_mov_b32_e32 v12, v37 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v21.h, s24 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v5, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v36.h, v5.h, s25 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v5.l, v5.h, s26 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v4.h, v36.h, s25 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, s26 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v36 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v20.l, v4.h, s27 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 0x8000, v4.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v20.h, s25 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v4, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v36.h, v4.h, s26 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 0x8000, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v4.h, s27 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v81, v81 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v3.h, v36.h, s26 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v4.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.h, v3.l, v19.l, s27 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v81, v81 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v19.l, v83.h, s28 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v3, v3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s41, 0x8000, v83.h +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v36.l +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v83, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.h, v2.l, v18.l, s28 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v1.l, v17.l, s40 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v32.h, v37.h, s44 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v36.h, v83.h, s27 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s40, 0x8000, v82.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v85, v85 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v18.h, s26 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v32.l, v1.l, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v83.h, s41 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v36.h, s28 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v3.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v31.h, v35.h, s42 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s42, 0x8000, v39.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 0x8000, v19.h +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v18.l, v82.h, s29 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 0x8000, v38.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v31.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v34.h, v39.h, s42 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v48.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v82, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v33.h, v38.h, s29 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v34.l, v1.h, s2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v37.l, v48.h, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v36.h, v82.h, s44 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v33.l, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v49.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v50.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v35.l, v1.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v82.h, s40 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v51.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v54.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v49.l, v50.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s45 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v1.l +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v39.l, v49.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v52.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v51.l, v51.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v53.h +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v17.l, v19.h, s27 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v38.l, v0.h, s4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v69.l, v52.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v55.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v48.l, v1.h, s5 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v36 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v69.h, v53.h, s2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v70.h, v55.h, s4 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v66.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v36.h, v19.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v65.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v50.l, v2.h, s6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v70.l, v54.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v52.l, v0.h, s7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v16.h, v19.h, s28 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v53.l, v1.h, s8 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v71.h, v65.h, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v54.l, v2.h, s9 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v67.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.h +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v64.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v65.l, v1.h, s12 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v67.h, s3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v55.l, v3.h, s10 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v16.l, v96.h, s41 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v71.l, v64.h, s0 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v80.l, v66.h, s2 +; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v16.h, v17.l, s4 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v96, v36 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v64.l, v0.h, s11 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v66.l, v16.l, s13 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v67.l, v15.l, s14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v68.l, v30.h, s15 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v36.h, v96.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v36.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v16.l, v96.h, s43 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v36.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.h, v16.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v36 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v17.h, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v32bf16: @@ -14669,34 +14424,29 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 { ; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee: ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_bf16_no_ieee: @@ -14732,40 +14482,34 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 { ; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l ; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v1.h, s0 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_bf16_no_ieee: @@ -15009,58 +14753,46 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> % ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v3, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v2.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16_no_ieee: @@ -15117,62 +14849,56 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> % ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v1.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v4.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, v4.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.l, s1 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2 -; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v1.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v4 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v3, v4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.l, v3.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v2.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v5.l, v0.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16_no_ieee: @@ -15521,66 +15247,67 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> % ; GFX11-TRUE16: ; %bb.0: ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v7, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v9, v11 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee: @@ -15652,77 +15379,80 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> % ; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v7, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v9, v11 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v5.l, v4.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.l, v0.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s2 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v5.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v5.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v3, v3 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v5 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v0.h, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s1 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v4.h, s2 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v1.l, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee: @@ -16183,85 +15913,83 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> % ; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v10, v8 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v9, v10 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v12 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v7.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v13 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 -; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v7.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v8.h +; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v8, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v0, v6 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.h +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v5.h +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee: @@ -16356,99 +16084,98 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> % ; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1 ; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0 -; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4 ; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8 -; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l ; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 -; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2 -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v10, v8 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l -; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v5.l, v4.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v9, v10 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v4.l, s1 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v15 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2 ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v7.l, v6.l, s2 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v12 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v7.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v3.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v6.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v13 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v5.l, s0 -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.l, v1.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8 +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l -; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v7.l, s2 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.l -; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v2.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v7.h ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v7 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v7.h, v6.h, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9 -; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v8.l, v0.h, s0 -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v2.h, s2 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.l, v3.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v8.h +; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v8, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v8.h, s0 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v8.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v7.h, s4 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v6.h, s0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v2.h, s0 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v0, v6 +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.h, v0.h, s2 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.h +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v0.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v5.h +; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff +; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v6.h, s2 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v5.h, s3 +; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v0.h, s1 ; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd -; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v0.l, vcc_lo +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo ; GFX12-TRUE16-NEXT: s_wait_alu 0xf1ff -; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s1 +; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2 ; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee: diff --git a/llvm/test/CodeGen/AMDGPU/move-load-addr-to-valu-flat.mir b/llvm/test/CodeGen/AMDGPU/move-load-addr-to-valu-flat.mir new file mode 100644 index 0000000..95ccf6c --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/move-load-addr-to-valu-flat.mir @@ -0,0 +1,357 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=si-fix-sgpr-copies -o - %s | FileCheck --check-prefix=GCN %s + +--- +name: flat_load_saddr_to_valu +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: flat_load_saddr_to_valu + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vreg_64_align2 = PHI [[COPY]], %bb.0, %7, %bb.1 + ; GCN-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[PHI]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub0 + ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub1 + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]], 1, implicit $exec + ; GCN-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], 0, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1 + ; GCN-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], 0, implicit $exec + ; GCN-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U64_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0_vgpr1 + %0:sreg_64 = COPY $vgpr0_vgpr1 + + bb.1: + %1:sreg_64_xexec_xnull = PHI %0, %bb.0, %2, %bb.1 + %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + %4:vgpr_32 = FLAT_LOAD_DWORD_SADDR %1, %3, 0, 0, implicit $exec, implicit $flat_scr + %2:sreg_64 = S_AND_B64 %1, 1, implicit-def $scc + S_CMP_LG_U64 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + +--- +name: flat_load_saddr_to_valu_non_zero_vaddr +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: flat_load_saddr_to_valu_non_zero_vaddr + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vreg_64_align2 = PHI [[COPY]], %bb.0, %7, %bb.1 + ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; GCN-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI]].sub0, implicit $exec + ; GCN-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI]].sub1, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1 + ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[REG_SEQUENCE]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub0 + ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub1 + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]], 1, implicit $exec + ; GCN-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], 0, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1 + ; GCN-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE1]], 0, implicit $exec + ; GCN-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U64_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0_vgpr1 + %0:sreg_64 = COPY $vgpr0_vgpr1 + + bb.1: + %1:sreg_64_xexec_xnull = PHI %0, %bb.0, %2, %bb.1 + %3:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + %4:vgpr_32 = FLAT_LOAD_DWORD_SADDR %1, %3, 0, 0, implicit $exec, implicit $flat_scr + %2:sreg_64 = S_AND_B64 %1, 1, implicit-def $scc + S_CMP_LG_U64 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + + +--- +name: flat_load_saddr_to_valu_undef_vaddr +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: flat_load_saddr_to_valu_undef_vaddr + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vreg_64_align2 = PHI [[COPY]], %bb.0, %7, %bb.1 + ; GCN-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI]].sub0, implicit $exec + ; GCN-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI]].sub1, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1 + ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[REG_SEQUENCE]], undef %4:vgpr_32, 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub0 + ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub1 + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]], 1, implicit $exec + ; GCN-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], 0, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1 + ; GCN-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE1]], 0, implicit $exec + ; GCN-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U64_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0_vgpr1 + %0:sreg_64 = COPY $vgpr0_vgpr1 + + bb.1: + %1:sreg_64_xexec_xnull = PHI %0, %bb.0, %2, %bb.1 + %4:vgpr_32 = FLAT_LOAD_DWORD_SADDR %1, undef %3:vgpr_32, 0, 0, implicit $exec, implicit $flat_scr + %2:sreg_64 = S_AND_B64 %1, 1, implicit-def $scc + S_CMP_LG_U64 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + +--- +name: flat_store_saddr_to_valu +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: flat_store_saddr_to_valu + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vreg_64_align2 = PHI [[COPY]], %bb.0, %7, %bb.1 + ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; GCN-NEXT: FLAT_STORE_DWORD [[PHI]], [[DEF]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub0 + ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub1 + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]], 1, implicit $exec + ; GCN-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], 0, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1 + ; GCN-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], 0, implicit $exec + ; GCN-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U64_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0_vgpr1 + %0:sreg_64 = COPY $vgpr0_vgpr1 + + bb.1: + %1:sreg_64_xexec_xnull = PHI %0, %bb.0, %2, %bb.1 + %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + %4:vgpr_32 = IMPLICIT_DEF + FLAT_STORE_DWORD_SADDR %3, %4, %1, 0, 0, implicit $exec, implicit $flat_scr + %2:sreg_64 = S_AND_B64 %1, 1, implicit-def $scc + S_CMP_LG_U64 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + +--- +name: flat_atomic_noret_saddr_to_valu +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: flat_atomic_noret_saddr_to_valu + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vreg_64_align2 = PHI [[COPY]], %bb.0, %6, %bb.1 + ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + ; GCN-NEXT: FLAT_ATOMIC_ADD [[PHI]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub0 + ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub1 + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]], 1, implicit $exec + ; GCN-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], 0, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1 + ; GCN-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], 0, implicit $exec + ; GCN-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U64_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0_vgpr1 + %0:sreg_64 = COPY $vgpr0_vgpr1 + + bb.1: + %1:sreg_64_xexec_xnull = PHI %0, %bb.0, %2, %bb.1 + %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + FLAT_ATOMIC_ADD_SADDR %3, %3, %1, 0, 0, implicit $exec, implicit $flat_scr + %2:sreg_64 = S_AND_B64 %1, 1, implicit-def $scc + S_CMP_LG_U64 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + +--- +name: flat_atomic_rtn_saddr_to_valu +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: flat_atomic_rtn_saddr_to_valu + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vreg_64_align2 = PHI [[COPY]], %bb.0, %7, %bb.1 + ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + ; GCN-NEXT: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[PHI]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub0 + ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[PHI]].sub1 + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]], 1, implicit $exec + ; GCN-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], 0, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1 + ; GCN-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], 0, implicit $exec + ; GCN-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U64_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0_vgpr1 + %0:sreg_64 = COPY $vgpr0_vgpr1 + + bb.1: + %1:sreg_64_xexec_xnull = PHI %0, %bb.0, %2, %bb.1 + %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + %4:vgpr_32 = FLAT_ATOMIC_ADD_SADDR_RTN %3, %3, %1, 0, 0, implicit $exec, implicit $flat_scr + %2:sreg_64 = S_AND_B64 %1, 1, implicit-def $scc + S_CMP_LG_U64 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + +--- +name: scratch_load_saddr_to_valu +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: scratch_load_saddr_to_valu + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %6, %bb.1 + ; GCN-NEXT: [[SCRATCH_LOAD_DWORD:%[0-9]+]]:vgpr_32 = SCRATCH_LOAD_DWORD [[PHI]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[PHI]], 1, implicit $exec + ; GCN-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U32_e64 [[V_AND_B32_e64_]], 0, implicit $exec + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_AND_B32_e64_]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U32_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0 + %0:sgpr_32 = COPY $vgpr0 + + bb.1: + %1:sgpr_32 = PHI %0, %bb.0, %2, %bb.1 + %4:vgpr_32 = SCRATCH_LOAD_DWORD_SADDR %1, 0, 0, implicit $exec, implicit $flat_scr + %2:sgpr_32 = S_AND_B32 %1, 1, implicit-def $scc + S_CMP_LG_U32 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... + +--- +name: scratch_store_saddr_to_valu +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: scratch_store_saddr_to_valu + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, %6, %bb.1 + ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; GCN-NEXT: SCRATCH_STORE_DWORD [[DEF]], [[PHI]], 0, 0, implicit $exec, implicit $flat_scr + ; GCN-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[PHI]], 1, implicit $exec + ; GCN-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U32_e64 [[V_AND_B32_e64_]], 0, implicit $exec + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_AND_B32_e64_]], implicit $exec + ; GCN-NEXT: $vcc_lo = S_AND_B32 $exec_lo, [[V_CMP_NE_U32_e64_]], implicit-def $scc + ; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc_lo + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0 + %0:sgpr_32 = COPY $vgpr0 + + bb.1: + %1:sgpr_32 = PHI %0, %bb.0, %2, %bb.1 + %4:vgpr_32 = IMPLICIT_DEF + SCRATCH_STORE_DWORD_SADDR %4, %1, 0, 0, implicit $exec, implicit $flat_scr + %2:sgpr_32 = S_AND_B32 %1, 1, implicit-def $scc + S_CMP_LG_U32 %2, 0, implicit-def $scc + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.2: + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll index 3844d60..cf244f0 100644 --- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll +++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll @@ -6,16 +6,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) { ; GFX942-LABEL: matmul_kernel: ; GFX942: ; %bb.0: ; %entry ; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 -; GFX942-NEXT: v_mov_b32_e32 v0, 0 -; GFX942-NEXT: v_accvgpr_write_b32 a2, v0 +; GFX942-NEXT: v_accvgpr_write_b32 a2, 0 ; GFX942-NEXT: s_mov_b32 s2, 0 ; GFX942-NEXT: v_accvgpr_write_b32 a1, 0 +; GFX942-NEXT: s_mov_b32 s3, 0 ; GFX942-NEXT: s_waitcnt lgkmcnt(0) ; GFX942-NEXT: s_cmp_lg_u32 s0, 0 ; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0 ; GFX942-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] ; GFX942-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 -; GFX942-NEXT: s_mov_b32 s3, 0 ; GFX942-NEXT: s_branch .LBB0_2 ; GFX942-NEXT: .LBB0_1: ; %bb2 ; GFX942-NEXT: ; in Loop: Header=BB0_2 Depth=1 @@ -43,16 +42,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) { ; GFX908-LABEL: matmul_kernel: ; GFX908: ; %bb.0: ; %entry ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 -; GFX908-NEXT: v_mov_b32_e32 v0, 0 +; GFX908-NEXT: v_accvgpr_write_b32 a2, 0 ; GFX908-NEXT: v_accvgpr_write_b32 a1, 0 ; GFX908-NEXT: s_mov_b32 s2, 0 -; GFX908-NEXT: v_accvgpr_write_b32 a2, v0 +; GFX908-NEXT: s_mov_b32 s3, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_cmp_lg_u32 s0, 0 ; GFX908-NEXT: s_cselect_b64 s[0:1], -1, 0 ; GFX908-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1] ; GFX908-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0 -; GFX908-NEXT: s_mov_b32 s3, 0 ; GFX908-NEXT: s_branch .LBB0_2 ; GFX908-NEXT: .LBB0_1: ; %bb2 ; GFX908-NEXT: ; in Loop: Header=BB0_2 Depth=1 diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir index ee54816..01506d0 100644 --- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir +++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir @@ -80,16 +80,16 @@ body: | ; COALESCE-NEXT: S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc ; COALESCE-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0 ; COALESCE-NEXT: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc - ; COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec + ; COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; COALESCE-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec ; COALESCE-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec - ; COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec + ; COALESCE-NEXT: undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0 ; COALESCE-NEXT: {{ $}} ; COALESCE-NEXT: bb.1: ; COALESCE-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000) ; COALESCE-NEXT: {{ $}} - ; COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0 + ; COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0 ; COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1 ; COALESCE-NEXT: $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc ; COALESCE-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc @@ -103,10 +103,10 @@ body: | ; COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc ; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0 ; COALESCE-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]] - ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]] - ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1 - ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1 - ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec + ; COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]] + ; COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1 + ; COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1 + ; COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec ; COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0 ; COALESCE-NEXT: {{ $}} ; COALESCE-NEXT: bb.3: @@ -134,16 +134,16 @@ body: | ; GFX908-COALESCE-NEXT: S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc ; GFX908-COALESCE-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0 ; GFX908-COALESCE-NEXT: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc - ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec + ; GFX908-COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX908-COALESCE-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec ; GFX908-COALESCE-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec - ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec + ; GFX908-COALESCE-NEXT: undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX908-COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0 ; GFX908-COALESCE-NEXT: {{ $}} ; GFX908-COALESCE-NEXT: bb.1: ; GFX908-COALESCE-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000) ; GFX908-COALESCE-NEXT: {{ $}} - ; GFX908-COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0 + ; GFX908-COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0 ; GFX908-COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1 ; GFX908-COALESCE-NEXT: $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc ; GFX908-COALESCE-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc @@ -157,10 +157,10 @@ body: | ; GFX908-COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc ; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0 ; GFX908-COALESCE-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]] - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]] - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1 - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1 - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec + ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]] + ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1 + ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1 + ; GFX908-COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec ; GFX908-COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0 ; GFX908-COALESCE-NEXT: {{ $}} ; GFX908-COALESCE-NEXT: bb.3: diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir index 49c0aaf..a9207de 100644 --- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir +++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir @@ -67,7 +67,7 @@ body: | ; COALESCE-NEXT: bb.1: ; COALESCE-NEXT: successors: %bb.3(0x80000000) ; COALESCE-NEXT: {{ $}} - ; COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec + ; COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; COALESCE-NEXT: S_BRANCH %bb.3 ; COALESCE-NEXT: {{ $}} ; COALESCE-NEXT: bb.2: @@ -78,13 +78,13 @@ body: | ; COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], 0, 0, 0, 0, implicit $mode, implicit $exec ; COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec ; COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec - ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec + ; COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec ; COALESCE-NEXT: {{ $}} ; COALESCE-NEXT: bb.3: - ; COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0 + ; COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0 ; COALESCE-NEXT: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec ; COALESCE-NEXT: undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec - ; COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec + ; COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0 ; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0 ; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0 @@ -105,28 +105,28 @@ body: | ; GFX908-COALESCE-NEXT: bb.1: ; GFX908-COALESCE-NEXT: successors: %bb.3(0x80000000) ; GFX908-COALESCE-NEXT: {{ $}} - ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec + ; GFX908-COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX908-COALESCE-NEXT: S_BRANCH %bb.3 ; GFX908-COALESCE-NEXT: {{ $}} ; GFX908-COALESCE-NEXT: bb.2: ; GFX908-COALESCE-NEXT: successors: %bb.3(0x80000000) ; GFX908-COALESCE-NEXT: {{ $}} - ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0 - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0 - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0 + ; GFX908-COALESCE-NEXT: undef [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[AV_MOV_1]].sub0 + ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub0 + ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub0 ; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0 ; GFX908-COALESCE-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0_sub1 - ; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec + ; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec ; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec ; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec - ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec + ; GFX908-COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec ; GFX908-COALESCE-NEXT: {{ $}} ; GFX908-COALESCE-NEXT: bb.3: - ; GFX908-COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0 + ; GFX908-COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0 ; GFX908-COALESCE-NEXT: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec ; GFX908-COALESCE-NEXT: undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec - ; GFX908-COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec + ; GFX908-COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec ; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0 ; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0 ; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0 diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll index 663fd98..ce96766 100644 --- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll +++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll @@ -17,9 +17,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 { ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) ; REGALLOC-GFX908-NEXT: [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 - ; REGALLOC-GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec - ; REGALLOC-GFX908-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec - ; REGALLOC-GFX908-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec + ; REGALLOC-GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; REGALLOC-GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec + ; REGALLOC-GFX908-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX908-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]] ; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) @@ -42,8 +42,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 { ; PEI-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; PEI-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) ; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec - ; PEI-GFX908-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec - ; PEI-GFX908-NEXT: renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec + ; PEI-GFX908-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; PEI-GFX908-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec ; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec ; PEI-GFX908-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5) ; PEI-GFX908-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1 @@ -62,9 +62,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 { ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) ; REGALLOC-GFX90A-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 - ; REGALLOC-GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec - ; REGALLOC-GFX90A-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec - ; REGALLOC-GFX90A-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec + ; REGALLOC-GFX90A-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; REGALLOC-GFX90A-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec + ; REGALLOC-GFX90A-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; REGALLOC-GFX90A-NEXT: S_ENDPGM 0 @@ -85,8 +85,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 { ; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) ; PEI-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec - ; PEI-GFX90A-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec - ; PEI-GFX90A-NEXT: renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec + ; PEI-GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; PEI-GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec ; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec ; PEI-GFX90A-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5) ; PEI-GFX90A-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1 diff --git a/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir b/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir index c2c5340..8145a1d7 100644 --- a/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir +++ b/llvm/test/CodeGen/AMDGPU/phi-vgpr-input-moveimm.mir @@ -167,3 +167,59 @@ body: | %1:sreg_32 = COPY %0 S_BRANCH %bb.2 ... + +--- + +name: phi_moveimm_av_pseudo_input +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: phi_moveimm_av_pseudo_input + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $sgpr0, $sgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 + ; GCN-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: successors: %bb.2(0x80000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc + ; GCN-NEXT: S_BRANCH %bb.2 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.2: + ; GCN-NEXT: successors: %bb.3(0x80000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[PHI:%[0-9]+]]:sreg_32 = PHI %5, %bb.3, [[S_ADD_U32_]], %bb.1 + ; GCN-NEXT: S_BRANCH %bb.3 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.3: + ; GCN-NEXT: successors: %bb.2(0x80000000) + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 + ; GCN-NEXT: S_BRANCH %bb.2 + bb.0: + successors: %bb.1 + liveins: $sgpr0, $sgpr1 + + %0:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec + + %4:sreg_32 = COPY $sgpr0 + %5:sreg_32 = COPY $sgpr1 + + bb.1: + successors: %bb.2 + %2:sreg_32 = S_ADD_U32 %4, %5, implicit-def $scc + S_BRANCH %bb.2 + + bb.2: + successors: %bb.3 + %3:sreg_32 = PHI %1, %bb.3, %2, %bb.1 + S_BRANCH %bb.3 + + bb.3: + successors: %bb.2 + %1:sreg_32 = COPY %0 + S_BRANCH %bb.2 +... diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll index 1ec9416..d48bfe0 100644 --- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll +++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll @@ -145,49 +145,29 @@ entry: ; Test skipping the lower-32-bit addition if it is unnecessary. define ptr @huge_offset_low_32_unused(ptr %p) { -; GFX942_PTRADD-LABEL: huge_offset_low_32_unused: -; GFX942_PTRADD: ; %bb.0: -; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX942_PTRADD-NEXT: s_mov_b32 s0, 0 -; GFX942_PTRADD-NEXT: s_mov_b32 s1, 1 -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1] -; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31] -; -; GFX942_LEGACY-LABEL: huge_offset_low_32_unused: -; GFX942_LEGACY: ; %bb.0: -; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX942_LEGACY-NEXT: v_add_u32_e32 v1, 1, v1 -; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31] +; GFX942-LABEL: huge_offset_low_32_unused: +; GFX942: ; %bb.0: +; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX942-NEXT: v_add_u32_e32 v1, 1, v1 +; GFX942-NEXT: s_setpc_b64 s[30:31] %gep = getelementptr inbounds i8, ptr %p, i64 u0x100000000 ret ptr %gep } ; Reassociate address computation if it leads to more scalar operations. define amdgpu_kernel void @reassoc_scalar_r(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) { -; GFX942_PTRADD-LABEL: reassoc_scalar_r: -; GFX942_PTRADD: ; %bb.0: ; %entry -; GFX942_PTRADD-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 -; GFX942_PTRADD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX942_PTRADD-NEXT: v_mov_b32_e32 v1, 0 -; GFX942_PTRADD-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942_PTRADD-NEXT: s_waitcnt lgkmcnt(0) -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[6:7] -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3] -; GFX942_PTRADD-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] -; GFX942_PTRADD-NEXT: s_endpgm -; -; GFX942_LEGACY-LABEL: reassoc_scalar_r: -; GFX942_LEGACY: ; %bb.0: ; %entry -; GFX942_LEGACY-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX942_LEGACY-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 -; GFX942_LEGACY-NEXT: v_mov_b32_e32 v1, 0 -; GFX942_LEGACY-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942_LEGACY-NEXT: s_waitcnt lgkmcnt(0) -; GFX942_LEGACY-NEXT: s_add_u32 s2, s2, s6 -; GFX942_LEGACY-NEXT: s_addc_u32 s3, s3, s7 -; GFX942_LEGACY-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1] -; GFX942_LEGACY-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] -; GFX942_LEGACY-NEXT: s_endpgm +; GFX942-LABEL: reassoc_scalar_r: +; GFX942: ; %bb.0: ; %entry +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_add_u32 s2, s2, s6 +; GFX942-NEXT: s_addc_u32 s3, s3, s7 +; GFX942-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1] +; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] +; GFX942-NEXT: s_endpgm entry: %voffset32 = call i32 @llvm.amdgcn.workitem.id.x() %voffset = zext i32 %voffset32 to i64 @@ -198,30 +178,18 @@ entry: } define amdgpu_kernel void @reassoc_scalar_l(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) { -; GFX942_PTRADD-LABEL: reassoc_scalar_l: -; GFX942_PTRADD: ; %bb.0: ; %entry -; GFX942_PTRADD-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 -; GFX942_PTRADD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX942_PTRADD-NEXT: v_mov_b32_e32 v1, 0 -; GFX942_PTRADD-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942_PTRADD-NEXT: s_waitcnt lgkmcnt(0) -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], s[6:7], 0, v[0:1] -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3] -; GFX942_PTRADD-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] -; GFX942_PTRADD-NEXT: s_endpgm -; -; GFX942_LEGACY-LABEL: reassoc_scalar_l: -; GFX942_LEGACY: ; %bb.0: ; %entry -; GFX942_LEGACY-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX942_LEGACY-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 -; GFX942_LEGACY-NEXT: v_mov_b32_e32 v1, 0 -; GFX942_LEGACY-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942_LEGACY-NEXT: s_waitcnt lgkmcnt(0) -; GFX942_LEGACY-NEXT: s_add_u32 s2, s2, s6 -; GFX942_LEGACY-NEXT: s_addc_u32 s3, s3, s7 -; GFX942_LEGACY-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1] -; GFX942_LEGACY-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] -; GFX942_LEGACY-NEXT: s_endpgm +; GFX942-LABEL: reassoc_scalar_l: +; GFX942: ; %bb.0: ; %entry +; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 +; GFX942-NEXT: v_mov_b32_e32 v1, 0 +; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: s_add_u32 s2, s2, s6 +; GFX942-NEXT: s_addc_u32 s3, s3, s7 +; GFX942-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1] +; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1] +; GFX942-NEXT: s_endpgm entry: %voffset32 = call i32 @llvm.amdgcn.workitem.id.x() %voffset = zext i32 %voffset32 to i64 @@ -233,24 +201,14 @@ entry: ; Tests the target-specific (ptradd x, shl(0 - y, k)) -> sub(x, shl(y, k)) fold define ptr addrspace(1) @shl_neg_offset(ptr addrspace(1) %p, i64 %noffset, i64 %shift) { -; GFX942_PTRADD-LABEL: shl_neg_offset: -; GFX942_PTRADD: ; %bb.0: -; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX942_PTRADD-NEXT: v_sub_co_u32_e32 v2, vcc, 0, v2 -; GFX942_PTRADD-NEXT: s_nop 1 -; GFX942_PTRADD-NEXT: v_subb_co_u32_e32 v3, vcc, 0, v3, vcc -; GFX942_PTRADD-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3] -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3] -; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31] -; -; GFX942_LEGACY-LABEL: shl_neg_offset: -; GFX942_LEGACY: ; %bb.0: -; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX942_LEGACY-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3] -; GFX942_LEGACY-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2 -; GFX942_LEGACY-NEXT: s_nop 1 -; GFX942_LEGACY-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc -; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31] +; GFX942-LABEL: shl_neg_offset: +; GFX942: ; %bb.0: +; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX942-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3] +; GFX942-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2 +; GFX942-NEXT: s_nop 1 +; GFX942-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc +; GFX942-NEXT: s_setpc_b64 s[30:31] %offset = sub i64 0, %noffset %x = shl i64 %offset, %shift %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 %x @@ -268,10 +226,9 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) { ; GFX942_PTRADD: ; %bb.0: ; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX942_PTRADD-NEXT: s_getpc_b64 s[0:1] -; GFX942_PTRADD-NEXT: s_add_u32 s0, s0, v0@rel32@lo+4 -; GFX942_PTRADD-NEXT: s_addc_u32 s1, s1, v0@rel32@hi+12 +; GFX942_PTRADD-NEXT: s_add_u32 s0, s0, v0@rel32@lo+14 +; GFX942_PTRADD-NEXT: s_addc_u32 s1, s1, v0@rel32@hi+22 ; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1] -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 10 ; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31] ; ; GFX942_LEGACY-LABEL: complextype_global_gep: @@ -291,30 +248,63 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) { ; Tests the tryFoldToMad64_32 PTRADD combine. define amdgpu_kernel void @fold_mad64(ptr addrspace(1) %p) { -; GFX942_PTRADD-LABEL: fold_mad64: -; GFX942_PTRADD: ; %bb.0: -; GFX942_PTRADD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 -; GFX942_PTRADD-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942_PTRADD-NEXT: v_mul_hi_u32_u24_e32 v1, 12, v0 -; GFX942_PTRADD-NEXT: v_mul_u32_u24_e32 v0, 12, v0 -; GFX942_PTRADD-NEXT: v_mov_b32_e32 v2, 1.0 -; GFX942_PTRADD-NEXT: s_waitcnt lgkmcnt(0) -; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1] -; GFX942_PTRADD-NEXT: global_store_dword v[0:1], v2, off -; GFX942_PTRADD-NEXT: s_endpgm -; -; GFX942_LEGACY-LABEL: fold_mad64: -; GFX942_LEGACY: ; %bb.0: -; GFX942_LEGACY-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 -; GFX942_LEGACY-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX942_LEGACY-NEXT: v_mov_b32_e32 v2, 1.0 -; GFX942_LEGACY-NEXT: s_waitcnt lgkmcnt(0) -; GFX942_LEGACY-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1] -; GFX942_LEGACY-NEXT: global_store_dword v[0:1], v2, off -; GFX942_LEGACY-NEXT: s_endpgm +; GFX942-LABEL: fold_mad64: +; GFX942: ; %bb.0: +; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX942-NEXT: v_mov_b32_e32 v2, 1.0 +; GFX942-NEXT: s_waitcnt lgkmcnt(0) +; GFX942-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1] +; GFX942-NEXT: global_store_dword v[0:1], v2, off +; GFX942-NEXT: s_endpgm %voffset32 = call i32 @llvm.amdgcn.workitem.id.x() %voffset = zext i32 %voffset32 to i64 %p1 = getelementptr inbounds %S, ptr addrspace(1) %p, i64 %voffset, i32 0 store float 1.0, ptr addrspace(1) %p1 ret void } + +; Use non-zero shift amounts in v_lshl_add_u64. +define ptr @select_v_lshl_add_u64(ptr %base, i64 %voffset) { +; GFX942_PTRADD-LABEL: select_v_lshl_add_u64: +; GFX942_PTRADD: ; %bb.0: +; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX942_PTRADD-NEXT: v_lshlrev_b64 v[2:3], 3, v[2:3] +; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3] +; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31] +; +; GFX942_LEGACY-LABEL: select_v_lshl_add_u64: +; GFX942_LEGACY: ; %bb.0: +; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX942_LEGACY-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 3, v[0:1] +; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31] + %gep = getelementptr inbounds i64, ptr %base, i64 %voffset + ret ptr %gep +} + +; Fold mul and add into v_mad, even if amdgpu-codegenprepare-mul24 turned the +; mul into a mul24. +define ptr @fold_mul24_into_mad(ptr %base, i64 %a, i64 %b) { +; GFX942_PTRADD-LABEL: fold_mul24_into_mad: +; GFX942_PTRADD: ; %bb.0: +; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX942_PTRADD-NEXT: v_and_b32_e32 v2, 0xfffff, v2 +; GFX942_PTRADD-NEXT: v_and_b32_e32 v4, 0xfffff, v4 +; GFX942_PTRADD-NEXT: v_mul_hi_u32_u24_e32 v3, v2, v4 +; GFX942_PTRADD-NEXT: v_mul_u32_u24_e32 v2, v2, v4 +; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3] +; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31] +; +; GFX942_LEGACY-LABEL: fold_mul24_into_mad: +; GFX942_LEGACY: ; %bb.0: +; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX942_LEGACY-NEXT: v_and_b32_e32 v2, 0xfffff, v2 +; GFX942_LEGACY-NEXT: v_and_b32_e32 v3, 0xfffff, v4 +; GFX942_LEGACY-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v3, v[0:1] +; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31] + %a_masked = and i64 %a, u0xfffff + %b_masked = and i64 %b, u0xfffff + %mul = mul i64 %a_masked, %b_masked + %gep = getelementptr inbounds i8, ptr %base, i64 %mul + ret ptr %gep +} diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir new file mode 100644 index 0000000..d27b4ea --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir @@ -0,0 +1,79 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5 +# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs --start-before=greedy,2 --stop-after=greedy,2 %s -o - | FileCheck %s + +# Make sure there's no machine verifier error + +# If RA is unable to find a register to allocate, then cleanupFailedVReg will do ad-hoc rewriting and will insert undefs to make the LiveRanges workable. +# %30:av_128 = COPY undef $vgpr0_vgpr1_vgpr2_vgpr3 is an example of such a rewrite / undef. If we were to want to spill %30, we should not be inserting +# actual spill code, as the source operand is undef. +# Check that there are no verfier issues with the LiveRange of $vgpr0_vgpr1_vgpr2_vgpr3 / that we do not insert spill code for %30. + + +--- | + define void @foo() #0 { + ret void + } + + attributes #0 = { "amdgpu-waves-per-eu"="8,8" } + +... + +--- +name: foo +tracksRegLiveness: true +stack: + - { id: 0, type: spill-slot, size: 32, alignment: 4 } +machineFunctionInfo: + maxKernArgAlign: 4 + isEntryFunction: true + waveLimiter: true + scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99' + stackPtrOffsetReg: '$sgpr32' + frameOffsetReg: '$sgpr33' + hasSpilledVGPRs: true + argumentInfo: + privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' } + dispatchPtr: { reg: '$sgpr4_sgpr5' } + kernargSegmentPtr: { reg: '$sgpr6_sgpr7' } + workGroupIDX: { reg: '$sgpr8' } + privateSegmentWaveByteOffset: { reg: '$sgpr9' } +body: | + bb.0: + ; CHECK-LABEL: name: foo + ; CHECK: INLINEASM &"; def $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 10 /* regdef */, def %10, 10 /* regdef */, def %1, 10 /* regdef */, def %2, 10 /* regdef */, def $vgpr0_vgpr1_vgpr2_vgpr3, 10 /* regdef */, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: KILL undef $vgpr0_vgpr1_vgpr2_vgpr3 + ; CHECK-NEXT: SI_SPILL_AV160_SAVE %2, %stack.1, $sgpr32, 0, implicit $exec :: (store (s160) into %stack.1, align 4, addrspace 5) + ; CHECK-NEXT: SI_SPILL_AV256_SAVE %1, %stack.3, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.3, align 4, addrspace 5) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_512 = COPY %10 + ; CHECK-NEXT: SI_SPILL_V512_SAVE [[COPY]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_512 = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: SI_SPILL_AV512_SAVE [[COPY1]], %stack.6, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.6, align 4, addrspace 5) + ; CHECK-NEXT: INLINEASM &"; clobber", 1 /* sideeffect attdialect */, 10 /* regdef */, implicit-def early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + ; CHECK-NEXT: [[SI_SPILL_AV512_RESTORE:%[0-9]+]]:av_512 = SI_SPILL_AV512_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.6, align 4, addrspace 5) + ; CHECK-NEXT: $agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16 = COPY [[SI_SPILL_AV512_RESTORE]] + ; CHECK-NEXT: [[SI_SPILL_V512_RESTORE:%[0-9]+]]:vreg_512 = SI_SPILL_V512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) + ; CHECK-NEXT: SI_SPILL_AV512_SAVE [[SI_SPILL_V512_RESTORE]], %stack.4, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.4, align 4, addrspace 5) + ; CHECK-NEXT: [[SI_SPILL_AV256_RESTORE:%[0-9]+]]:av_256 = SI_SPILL_AV256_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.3, align 4, addrspace 5) + ; CHECK-NEXT: SI_SPILL_AV256_SAVE [[SI_SPILL_AV256_RESTORE]], %stack.5, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.5, align 4, addrspace 5) + ; CHECK-NEXT: [[SI_SPILL_AV160_RESTORE:%[0-9]+]]:vreg_160 = SI_SPILL_AV160_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s160) from %stack.1, align 4, addrspace 5) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) + ; CHECK-NEXT: [[SI_SPILL_AV512_RESTORE1:%[0-9]+]]:av_512 = SI_SPILL_AV512_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.4, align 4, addrspace 5) + ; CHECK-NEXT: [[SI_SPILL_AV256_RESTORE1:%[0-9]+]]:av_256 = SI_SPILL_AV256_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.5, align 4, addrspace 5) + ; CHECK-NEXT: INLINEASM &"; use $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 9 /* reguse */, [[SI_SPILL_AV512_RESTORE1]], 9 /* reguse */, [[SI_SPILL_AV256_RESTORE1]], 9 /* reguse */, [[SI_SPILL_AV160_RESTORE]], 9 /* reguse */, undef $vgpr0_vgpr1_vgpr2_vgpr3, 9 /* reguse */, $agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16 + ; CHECK-NEXT: SI_RETURN + INLINEASM &"; def $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 10, def %22:vreg_512, 10, def %25:vreg_256, 10, def %28:vreg_160, 10, def $vgpr0_vgpr1_vgpr2_vgpr3, 10, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + %30:av_128 = COPY undef $vgpr0_vgpr1_vgpr2_vgpr3 + %27:av_160 = COPY %28:vreg_160 + %24:av_256 = COPY %25:vreg_256 + SI_SPILL_V512_SAVE %22:vreg_512, %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) + %18:vreg_512 = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + INLINEASM &"; clobber", 1 /* sideeffect attdialect */, 10, implicit-def early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 + $agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16 = COPY %18:vreg_512 + %23:vreg_512 = SI_SPILL_V512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) + %26:vreg_256 = COPY %24:av_256 + %29:vreg_160 = COPY %27:av_160 + $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %30:av_128 + INLINEASM &"; use $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 9, %23:vreg_512, 9, %26:vreg_256, 9, %29:vreg_160, 9, undef $vgpr0_vgpr1_vgpr2_vgpr3, 9, $agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16 + SI_RETURN + +... diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll index ba9dd8f..5d0e4bf 100644 --- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll +++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll @@ -559,16 +559,19 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 ; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[12:13] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[12:13] ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[12:13] ; GFX9-O0-NEXT: v_mov_b32_e32 v3, s10 ; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[8:9] ; GFX9-O0-NEXT: ; implicit-def: $sgpr8 @@ -1943,16 +1946,19 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) { ; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6 ; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v1, v4, s[12:13] +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, s10 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[12:13] ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; implicit-def: $sgpr12 ; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[8:9] ; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11 -; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[8:9] +; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v3, v4, s[12:13] ; GFX9-O0-NEXT: v_mov_b32_e32 v3, s10 ; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[8:9] ; GFX9-O0-NEXT: ; implicit-def: $sgpr8 diff --git a/llvm/test/CodeGen/AMDGPU/select-cmp-shared-constant-fp.ll b/llvm/test/CodeGen/AMDGPU/select-cmp-shared-constant-fp.ll new file mode 100644 index 0000000..11af704 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/select-cmp-shared-constant-fp.ll @@ -0,0 +1,1429 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX900 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX1010 %s + +; Test the CMP+SELECT optimization that folds shared constants to reduce +; register pressure. + +;------------------------------------------------------------------------------ +; F32 Tests +;------------------------------------------------------------------------------ + +; Should be folded: fcmp oeq + select with constant in true value +define float @fcmp_select_fold_oeq_f32_imm(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_f32_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x42487ed8 +; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_f32_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0x42487ed8, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 0x40490FDB00000000 + %sel = select i1 %cmp, float 0x40490FDB00000000, float %other + ret float %sel +} + +; Should be folded: fcmp oeq + select with constant in true value (commutative) +define float @fcmp_select_fold_oeq_imm_f32(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_imm_f32: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x42487ed8 +; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_imm_f32: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0x42487ed8, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float 0x40490FDB00000000, %arg + %sel = select i1 %cmp, float 0x40490FDB00000000, float %other + ret float %sel +} + +; Should be folded: fcmp one + select with constant in false value +define float @fcmp_select_fold_one_f32_imm(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_fold_one_f32_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x402df850 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_f32_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x402df850, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one float %arg, 0x4005BF0A00000000 + %sel = select i1 %cmp, float %other, float 0x4005BF0A00000000 + ret float %sel +} + +; Should be folded: fcmp one + select with constant in false value (commutative) +define float @fcmp_select_fold_one_imm_f32(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_fold_one_imm_f32: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x402df850 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_imm_f32: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x402df850, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one float 0x4005BF0A00000000, %arg + %sel = select i1 %cmp, float %other, float 0x4005BF0A00000000 + ret float %sel +} + +; Should NOT be folded: different constants +define float @fcmp_select_no_fold_f32_different_const(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_f32_different_const: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x42487ed8 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x46487ed8 +; GFX900-NEXT: v_cmp_neq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f32_different_const: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0x42487ed8, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x46487ed8, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 0x40490FDB00000000 + %sel = select i1 %cmp, float 0x40C90FDB00000000, float %other + ret float %sel +} + +; Should NOT be folded: fcmp oeq with constant in other position +define float @fcmp_select_no_fold_f32_other_pos(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_f32_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x42487ed8 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x42487ed8 +; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f32_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0x42487ed8, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x42487ed8, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 0x40490FDB00000000 + %sel = select i1 %cmp, float %other, float 0x40490FDB00000000 + ret float %sel +} + +; Should NOT be folded: unsupported comparison type +define float @fcmp_select_no_fold_f32_unsupported_cmp(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_f32_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x42487ed8 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x42487ed8 +; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f32_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x42487ed8, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x42487ed8, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp olt float %arg, 0x40490FDB00000000 + %sel = select i1 %cmp, float %other, float 0x40490FDB00000000 + ret float %sel +} + +; Should NOT be folded: imm can be encoded into cndmask +define float @fcmp_select_no_fold_f32_enc_imm(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_f32_enc_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_neq_f32_e32 vcc, 1.0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 1.0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f32_enc_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f32_e32 vcc_lo, 1.0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 1.0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 1.0 + %sel = select i1 %cmp, float 1.0, float %other + ret float %sel +} + +; Should NOT be folded: imm can be encoded into cndmask +define float @fcmp_select_no_fold_f32_enc_imm_2(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_f32_enc_imm_2: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, -4.0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, -4.0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f32_enc_imm_2: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, -4.0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, -4.0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one float -4.0, %arg + %sel = select i1 %cmp, float %other, float -4.0 + ret float %sel +} + +; Should NOT be folded: fcmp oeq with zero constant +define float @fcmp_select_no_fold_oeq_f32_zero(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_oeq_f32_zero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_neq_f32_e32 vcc, 0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_oeq_f32_zero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 0.0 + %sel = select i1 %cmp, float 0.0, float %other + ret float %sel +} + +; Should NOT be folded: fcmp one with negative zero constant +define float @fcmp_select_no_fold_one_f32_negzero(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_one_f32_negzero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_brev_b32 s4, 1 +; GFX900-NEXT: v_bfrev_b32_e32 v2, 1 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_one_f32_negzero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x80000000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x80000000, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one float -0.0, %arg ; 0x8000000000000000 + %sel = select i1 %cmp, float %other, float -0.0 ;0x8000000000000000 + ret float %sel +} + +; NaN values should bypass the optimization due to special IEEE 754 behavior +; fcmp oeq with NaN always returns false, so select always chooses %other +define float @fcmp_select_no_fold_oeq_f32_nan(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_oeq_f32_nan: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, v1 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_oeq_f32_nan: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, v1 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 0x7FF8000000000000 + %sel = select i1 %cmp, float 0x7FF8000000000000, float %other + ret float %sel +} + +; NaN values should bypass the optimization due to special IEEE 754 behavior +; fcmp one with NaN always returns false, so select always chooses the NaN constant +define float @fcmp_select_no_fold_one_f32_nan(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_one_f32_nan: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, 0x7fc00000 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_one_f32_nan: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, 0x7fc00000 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one float 0x7FF8000000000000, %arg + %sel = select i1 %cmp, float %other, float 0x7FF8000000000000 + ret float %sel +} + +; Should NOT be folded: fcmp one with positive infinity +; Infinity values should bypass the optimization, generating unfolded code +define float @fcmp_select_no_fold_posinf_oeq_f32(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_posinf_oeq_f32: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x7f800000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f800000 +; GFX900-NEXT: v_cmp_neq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_posinf_oeq_f32: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0x7f800000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq float %arg, 0x7FF0000000000000 + %sel = select i1 %cmp, float 0x7FF0000000000000, float %other + ret float %sel +} + +; Should NOT be folded: fcmp one with negative infinity +; Infinity values should bypass the optimization, generating unfolded code +define float @fcmp_select_no_fold_neginf_f32_one(float %arg, float %other) { +; GFX900-LABEL: fcmp_select_no_fold_neginf_f32_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xff800000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0xff800000 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_neginf_f32_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0xff800000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0xff800000, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one float 0xFFF0000000000000, %arg + %sel = select i1 %cmp, float %other, float 0xFFF0000000000000 + ret float %sel +} + +;------------------------------------------------------------------------------ +; F64 Tests +;------------------------------------------------------------------------------ + +; Should be folded: f64 fcmp oeq + select with constant in true value +define double @fcmp_select_fold_oeq_f64_imm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_f64_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX900-NEXT: s_mov_b32 s5, 0x400921fb +; GFX900-NEXT: v_cmp_eq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_f64_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX1010-NEXT: s_mov_b32 s5, 0x400921fb +; GFX1010-NEXT: v_cmp_eq_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 3.141592653589793 + %sel = select i1 %cmp, double 3.141592653589793, double %other + ret double %sel +} +; Should be folded: f64 fcmp oeq + select with constant in true value (commutative) +define double @fcmp_select_fold_oeq_imm_f64(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_imm_f64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX900-NEXT: s_mov_b32 s5, 0x400921fb +; GFX900-NEXT: v_cmp_eq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_imm_f64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX1010-NEXT: s_mov_b32 s5, 0x400921fb +; GFX1010-NEXT: v_cmp_eq_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double 3.141592653589793, %arg + %sel = select i1 %cmp, double 3.141592653589793, double %other + ret double %sel +} + +; Should be folded: f64 fcmp one + select with constant in false value +define double @fcmp_select_fold_one_f64_imm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_fold_one_f64_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x8b145769 +; GFX900-NEXT: s_mov_b32 s5, 0x4005bf0a +; GFX900-NEXT: v_cmp_lg_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_f64_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x8b145769 +; GFX1010-NEXT: s_mov_b32 s5, 0x4005bf0a +; GFX1010-NEXT: v_cmp_lg_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one double %arg, 2.718281828459045 + %sel = select i1 %cmp, double %other, double 2.718281828459045 + ret double %sel +} +; Should be folded: f64 fcmp one + select with constant in false value (commutative) +define double @fcmp_select_fold_one_imm_f64(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_fold_one_imm_f64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x8b145769 +; GFX900-NEXT: s_mov_b32 s5, 0x4005bf0a +; GFX900-NEXT: v_cmp_lg_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_imm_f64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x8b145769 +; GFX1010-NEXT: s_mov_b32 s5, 0x4005bf0a +; GFX1010-NEXT: v_cmp_lg_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one double 2.718281828459045, %arg + %sel = select i1 %cmp, double %other, double 2.718281828459045 + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with constant in other position +define double @fcmp_select_no_fold_f64_other_pos(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_f64_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX900-NEXT: s_mov_b32 s5, 0x400921fb +; GFX900-NEXT: v_cmp_eq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v4, 0x54442d18 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x400921fb +; GFX900-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f64_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX1010-NEXT: s_mov_b32 s5, 0x400921fb +; GFX1010-NEXT: v_cmp_eq_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x54442d18, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x400921fb, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 3.141592653589793 + %sel = select i1 %cmp, double %other, double 3.141592653589793 + ret double %sel +} + +; Should NOT be folded: f64 fcmp unsupported comparison type +define double @fcmp_select_no_fold_f64_unsupported_cmp(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_f64_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX900-NEXT: s_mov_b32 s5, 0x400921fb +; GFX900-NEXT: v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v4, 0x54442d18 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x400921fb +; GFX900-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f64_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX1010-NEXT: s_mov_b32 s5, 0x400921fb +; GFX1010-NEXT: v_cmp_gt_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x54442d18, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x400921fb, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp olt double %arg, 3.141592653589793 + %sel = select i1 %cmp, double %other, double 3.141592653589793 + ret double %sel +} + +; Should NOT be folded: imm can be encoded into cndmask +define double @fcmp_select_no_fold_f64_enc_imm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_f64_enc_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, 1.0, v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v1, 0x3ff00000 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f64_enc_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, 1.0, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x3ff00000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 1.0 + %sel = select i1 %cmp, double 1.0, double %other + ret double %sel +} + +; Should NOT be folded: imm can be encoded into cndmask +define double @fcmp_select_no_fold_f64_enc_imm_2(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_f64_enc_imm_2: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_lg_f64_e32 vcc, -4.0, v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v1, 0xc0100000 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f64_enc_imm_2: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f64_e32 vcc_lo, -4.0, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0xc0100000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one double -4.0, %arg + %sel = select i1 %cmp, double %other, double -4.0 + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with zero constant +define double @fcmp_select_no_fold_oeq_f64_zero(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_oeq_f64_zero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, 0, v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_oeq_f64_zero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, 0, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 0.0 + %sel = select i1 %cmp, double 0.0, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp one with negative zero constant +define double @fcmp_select_no_fold_one_f64_negzero(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_one_f64_negzero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0 +; GFX900-NEXT: s_brev_b32 s5, 1 +; GFX900-NEXT: v_cmp_lg_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_bfrev_b32_e32 v1, 1 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_one_f64_negzero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f64_e32 vcc_lo, 0x80000000, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x80000000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one double -0.0, %arg + %sel = select i1 %cmp, double %other, double -0.0 + ret double %sel +} + +; Should NOT be folded: f64 different constants +define double @fcmp_select_no_fold_f64_different_const(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_f64_different_const: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX900-NEXT: s_mov_b32 s5, 0x400921fb +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v4, 0x8b145769 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x4005bf0a +; GFX900-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f64_different_const: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0x54442d18 +; GFX1010-NEXT: s_mov_b32 s5, 0x400921fb +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x8b145769, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x4005bf0a, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 3.141592653589793 + %sel = select i1 %cmp, double 2.718281828459045, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with NaN constant +; fcmp oeq with NaN always returns false, so select always chooses %other +define double @fcmp_select_no_fold_nan_f64(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_f64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v1, v3 +; GFX900-NEXT: v_mov_b32_e32 v0, v2 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_f64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v1, v3 +; GFX1010-NEXT: v_mov_b32_e32 v0, v2 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 0x7FF8000000000000 + %sel = select i1 %cmp, double 0x7FF8000000000000, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with NaN constant (commutative variant) +; fcmp oeq with NaN always returns false, so select always chooses %other +define double @fcmp_select_no_fold_nan_f64_comm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_f64_comm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v1, v3 +; GFX900-NEXT: v_mov_b32_e32 v0, v2 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_f64_comm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v1, v3 +; GFX1010-NEXT: v_mov_b32_e32 v0, v2 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double 0x7FF8000000000000, %arg + %sel = select i1 %cmp, double 0x7FF8000000000000, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp one with NaN constant +; fcmp one with NaN always returns false, so select always chooses the NaN constant +define double @fcmp_select_no_fold_nan_f64_one(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_f64_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, 0 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x7ff80000 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_f64_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, 0 +; GFX1010-NEXT: v_mov_b32_e32 v1, 0x7ff80000 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one double %arg, 0x7FF8000000000000 + %sel = select i1 %cmp, double %other, double 0x7FF8000000000000 + ret double %sel +} + +; Should NOT be folded: f64 fcmp one with NaN constant (commutative variant) +; fcmp one with NaN always returns false, so select always chooses the NaN constant +define double @fcmp_select_no_fold_nan_f64_one_comm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_f64_one_comm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, 0 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x7ff80000 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_f64_one_comm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, 0 +; GFX1010-NEXT: v_mov_b32_e32 v1, 0x7ff80000 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one double 0x7FF8000000000000, %arg + %sel = select i1 %cmp, double %other, double 0x7FF8000000000000 + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with positive infinity +; Infinity values should bypass the optimization, generating unfolded code +define double @fcmp_select_no_fold_posinf_f64(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_posinf_f64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0 +; GFX900-NEXT: s_mov_b32 s5, 0x7ff00000 +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v1, 0x7ff00000 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_posinf_f64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, 0x7ff00000, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x7ff00000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 0x7FF0000000000000 + %sel = select i1 %cmp, double 0x7FF0000000000000, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with negative infinity +; Infinity values should bypass the optimization, generating unfolded code +define double @fcmp_select_no_fold_neginf_f64(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_neginf_f64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0 +; GFX900-NEXT: s_mov_b32 s5, 0xfff00000 +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v1, 0xfff00000 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_neginf_f64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, 0xfff00000, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0xfff00000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double %arg, 0xFFF0000000000000 + %sel = select i1 %cmp, double 0xFFF0000000000000, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with positive infinity (commutative variant) +; Infinity values should bypass the optimization, generating unfolded code +define double @fcmp_select_no_fold_posinf_f64_comm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_posinf_f64_comm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0 +; GFX900-NEXT: s_mov_b32 s5, 0x7ff00000 +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v1, 0x7ff00000 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_posinf_f64_comm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, 0x7ff00000, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x7ff00000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double 0x7FF0000000000000, %arg + %sel = select i1 %cmp, double 0x7FF0000000000000, double %other + ret double %sel +} + +; Should NOT be folded: f64 fcmp oeq with negative infinity (commutative variant) +; Infinity values should bypass the optimization, generating unfolded code +define double @fcmp_select_no_fold_neginf_f64_comm(double %arg, double %other) { +; GFX900-LABEL: fcmp_select_no_fold_neginf_f64_comm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0 +; GFX900-NEXT: s_mov_b32 s5, 0xfff00000 +; GFX900-NEXT: v_cmp_neq_f64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v1, 0xfff00000 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_neginf_f64_comm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f64_e32 vcc_lo, 0xfff00000, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0xfff00000, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq double 0xFFF0000000000000, %arg + %sel = select i1 %cmp, double 0xFFF0000000000000, double %other + ret double %sel +} + +;------------------------------------------------------------------------------ +; F16 Tests +;------------------------------------------------------------------------------ + +; Should be folded: f16 fcmp oeq + select with constant in true value +define half @fcmp_select_fold_oeq_f16_imm(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_f16_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4248 +; GFX900-NEXT: v_cmp_eq_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_f16_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0x4248, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq half %arg, 0xH4248 + %sel = select i1 %cmp, half 0xH4248, half %other + ret half %sel +} + +; Should be folded: f16 fcmp oeq + select with constant in true value (commutative) +define half @fcmp_select_fold_oeq_imm_f16(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_imm_f16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4248 +; GFX900-NEXT: v_cmp_eq_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_imm_f16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0x4248, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq half 0xH4248, %arg + %sel = select i1 %cmp, half 0xH4248, half %other + ret half %sel +} + +; Should be folded: f16 fcmp one + select with constant in false value +define half @fcmp_select_fold_one_f16_imm(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_fold_one_f16_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4020 +; GFX900-NEXT: v_cmp_lg_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_f16_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0x4020, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one half %arg, 0xH4020 + %sel = select i1 %cmp, half %other, half 0xH4020 + ret half %sel +} + +; Should be folded: f16 fcmp one + select with constant in false value (commutative) +define half @fcmp_select_fold_one_imm_f16(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_fold_one_imm_f16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4020 +; GFX900-NEXT: v_cmp_lg_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_imm_f16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0x4020, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one half 0xH4020, %arg + %sel = select i1 %cmp, half %other, half 0xH4020 + ret half %sel +} + +; Should NOT be folded: different constants +define half @fcmp_select_no_fold_f16_different_const(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_f16_different_const: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4248 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x4300 +; GFX900-NEXT: v_cmp_neq_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f16_different_const: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f16_e32 vcc_lo, 0x4248, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x4300, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq half %arg, 0xH4248 + %sel = select i1 %cmp, half 0xH4300, half %other + ret half %sel +} + +; Should NOT be folded: NaN values bypass optimization +define half @fcmp_select_no_fold_nan_f16(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_f16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, v1 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_f16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, v1 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq half %arg, 0xH7e00 + %sel = select i1 %cmp, half 0xH7e00, half %other + ret half %sel +} + +; Should NOT be folded: f16 fcmp one with NaN constant +define half @fcmp_select_no_fold_nan_f16_one(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_f16_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, 0x7e00 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_f16_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, 0x7e00 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one half %arg, 0xH7e00 + %sel = select i1 %cmp, half %other, half 0xH7e00 + ret half %sel +} + +; Should NOT be folded: f16 fcmp one with +Inf constant +define half @fcmp_select_no_fold_posinf_f16_one(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_posinf_f16_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7c00 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x7c00 +; GFX900-NEXT: v_cmp_lg_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_posinf_f16_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0x7c00, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one half %arg, 0xH7c00 + %sel = select i1 %cmp, half %other, half 0xH7c00 + ret half %sel +} + +; Should NOT be folded: f16 fcmp one with -Inf constant +define half @fcmp_select_no_fold_neginf_f16_one(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_neginf_f16_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xfc00 +; GFX900-NEXT: v_mov_b32_e32 v2, 0xfc00 +; GFX900-NEXT: v_cmp_lg_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_neginf_f16_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0xfc00, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0xfc00, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one half %arg, 0xHfc00 + %sel = select i1 %cmp, half %other, half 0xHfc00 + ret half %sel +} +; Should NOT be folded: f16 fcmp oeq with zero constant +define half @fcmp_select_no_fold_oeq_f16_zero(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_oeq_f16_zero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_neq_f16_e32 vcc, 0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_oeq_f16_zero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_neq_f16_e32 vcc_lo, 0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq half %arg, 0xH0000 + %sel = select i1 %cmp, half 0xH0000, half %other + ret half %sel +} +; Should NOT be folded: f16 fcmp one with negative zero constant +define half @fcmp_select_no_fold_one_f16_negzero(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_one_f16_negzero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0x8000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x8000 +; GFX900-NEXT: v_cmp_lg_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_one_f16_negzero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0x8000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x8000, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one half 0xH8000, %arg + %sel = select i1 %cmp, half %other, half 0xH8000 + ret half %sel +} + +; Should NOT be folded: f16 fcmp oeq with constant in other position +define half @fcmp_select_no_fold_f16_other_pos(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_f16_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4248 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x4248 +; GFX900-NEXT: v_cmp_eq_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f16_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0x4248, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x4248, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq half %arg, 0xH4248 + %sel = select i1 %cmp, half %other, half 0xH4248 + ret half %sel +} + +; Should NOT be folded: f16 unsupported comparison type +define half @fcmp_select_no_fold_f16_unsupported_cmp(half %arg, half %other) { +; GFX900-LABEL: fcmp_select_no_fold_f16_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x4248 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x4248 +; GFX900-NEXT: v_cmp_gt_f16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_f16_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0x4248, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x4248, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp olt half %arg, 0xH4248 + %sel = select i1 %cmp, half %other, half 0xH4248 + ret half %sel +} + +;------------------------------------------------------------------------------ +; BF16 Tests +;------------------------------------------------------------------------------ + +; Should be folded: bfloat fcmp oeq + select with constant in true value +define bfloat @fcmp_select_fold_oeq_bf16_imm(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_bf16_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x42480000 +; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, s4, v2 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_bf16_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX1010-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0x42480000, v2 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq bfloat %arg, 0xR4248 + %sel = select i1 %cmp, bfloat 0xR4248, bfloat %other + ret bfloat %sel +} + +; Should be folded: bfloat fcmp oeq + select with constant in true value (commutative) +define bfloat @fcmp_select_fold_oeq_imm_bf16(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_fold_oeq_imm_bf16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x42480000 +; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, s4, v2 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_oeq_imm_bf16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX1010-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0x42480000, v2 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq bfloat 0xR4248, %arg + %sel = select i1 %cmp, bfloat 0xR4248, bfloat %other + ret bfloat %sel +} + +; Should be folded: bfloat fcmp one + select with constant in false value +define bfloat @fcmp_select_fold_one_bf16_imm(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_fold_one_bf16_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x40200000 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v2 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_bf16_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x40200000, v2 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one bfloat %arg, 0xR4020 + %sel = select i1 %cmp, bfloat %other, bfloat 0xR4020 + ret bfloat %sel +} + +; Should be folded: bfloat fcmp one + select with constant in false value (commutative) +define bfloat @fcmp_select_fold_one_imm_bf16(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_fold_one_imm_bf16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x40200000 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v2 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_fold_one_imm_bf16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x40200000, v2 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one bfloat 0xR4020, %arg + %sel = select i1 %cmp, bfloat %other, bfloat 0xR4020 + ret bfloat %sel +} + +; Should NOT be folded: different constants +define bfloat @fcmp_select_no_fold_bf16_different_const(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_bf16_different_const: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x42480000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x4300 +; GFX900-NEXT: v_cmp_neq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_bf16_different_const: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0x42480000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x4300, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq bfloat %arg, 0xR4248 + %sel = select i1 %cmp, bfloat 0xR4300, bfloat %other + ret bfloat %sel +} + +; Should NOT be folded: NaN values bypass optimization +define bfloat @fcmp_select_no_fold_nan_bf16(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_bf16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, v1 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_bf16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, v1 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq bfloat %arg, 0xR7FC0 + %sel = select i1 %cmp, bfloat 0xR7FC0, bfloat %other + ret bfloat %sel +} + +; Should NOT be folded: bfloat fcmp one with NaN constant +define bfloat @fcmp_select_no_fold_nan_bf16_one(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_nan_bf16_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v0, 0x7fc0 +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_nan_bf16_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v0, 0x7fc0 +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one bfloat %arg, 0xR7FC0 + %sel = select i1 %cmp, bfloat %other, bfloat 0xR7FC0 + ret bfloat %sel +} + +; Should NOT be folded: bfloat fcmp one with +Inf constant +define bfloat @fcmp_select_no_fold_posinf_bf16_one(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_posinf_bf16_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x7f800000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x7f80 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_posinf_bf16_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x7f800000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x7f80, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one bfloat %arg, 0xR7F80 + %sel = select i1 %cmp, bfloat %other, bfloat 0xR7F80 + ret bfloat %sel +} + +; Should NOT be folded: bfloat fcmp one with -Inf constant +define bfloat @fcmp_select_no_fold_neginf_bf16_one(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_neginf_bf16_one: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0xff800000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0xffffff80 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_neginf_bf16_one: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0xff800000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0xffffff80, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one bfloat %arg, 0xRFF80 + %sel = select i1 %cmp, bfloat %other, bfloat 0xRFF80 + ret bfloat %sel +} + +; Should NOT be folded: bfloat fcmp oeq with zero constant +define bfloat @fcmp_select_no_fold_oeq_bf16_zero(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_oeq_bf16_zero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: v_cmp_neq_f32_e32 vcc, 0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_oeq_bf16_zero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq bfloat %arg, 0xR0000 + %sel = select i1 %cmp, bfloat 0xR0000, bfloat %other + ret bfloat %sel +} + +; Should NOT be folded: bfloat fcmp one with negative zero constant +define bfloat @fcmp_select_no_fold_one_bf16_negzero(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_one_bf16_negzero: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_brev_b32 s4, 1 +; GFX900-NEXT: v_mov_b32_e32 v2, 0xffff8000 +; GFX900-NEXT: v_cmp_lg_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_one_bf16_negzero: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_lg_f32_e32 vcc_lo, 0x80000000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0xffff8000, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp one bfloat 0xR8000, %arg + %sel = select i1 %cmp, bfloat %other, bfloat 0xR8000 + ret bfloat %sel +} + +; Should NOT be folded: bfloat fcmp oeq with constant in other position +define bfloat @fcmp_select_no_fold_bf16_other_pos(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_bf16_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x42480000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x4248 +; GFX900-NEXT: v_cmp_eq_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_bf16_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0x42480000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x4248, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp oeq bfloat %arg, 0xR4248 + %sel = select i1 %cmp, bfloat %other, bfloat 0xR4248 + ret bfloat %sel +} + +; Should NOT be folded: bfloat unsupported comparison type +define bfloat @fcmp_select_no_fold_bf16_unsupported_cmp(bfloat %arg, bfloat %other) { +; GFX900-LABEL: fcmp_select_no_fold_bf16_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX900-NEXT: s_mov_b32 s4, 0x42480000 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x4248 +; GFX900-NEXT: v_cmp_gt_f32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: fcmp_select_no_fold_bf16_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX1010-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x42480000, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x4248, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = fcmp olt bfloat %arg, 0xR4248 + %sel = select i1 %cmp, bfloat %other, bfloat 0xR4248 + ret bfloat %sel +} diff --git a/llvm/test/CodeGen/AMDGPU/select-cmp-shared-constant-int.ll b/llvm/test/CodeGen/AMDGPU/select-cmp-shared-constant-int.ll new file mode 100644 index 0000000..4383cfd --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/select-cmp-shared-constant-int.ll @@ -0,0 +1,955 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX900 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX1010 %s + +;------------------------------------------------------------------------------ +; I32 Tests +;------------------------------------------------------------------------------ + +; Should be folded: icmp eq + select with constant in true value +define i32 @icmp_select_fold_eq_i32_imm(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_fold_eq_i32_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_i32_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i32 %arg, 4242 + %sel = select i1 %cmp, i32 4242, i32 %other + ret i32 %sel +} + +; Should be folded: icmp eq + select with constant in true value (commutative) +define i32 @icmp_select_fold_eq_imm_i32(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_fold_eq_imm_i32: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_imm_i32: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i32 4242, %arg + %sel = select i1 %cmp, i32 4242, i32 %other + ret i32 %sel +} + +; Should be folded: icmp ne + select with constant in false value +define i32 @icmp_select_fold_ne_i32_imm(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_fold_ne_i32_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_i32_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i32 %arg, 4242 + %sel = select i1 %cmp, i32 %other, i32 4242 + ret i32 %sel +} + +; Should be folded: icmp ne + select with constant in false value (commutative) +define i32 @icmp_select_fold_ne_imm_i32(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_fold_ne_imm_i32: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_imm_i32: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i32 4242, %arg + %sel = select i1 %cmp, i32 %other, i32 4242 + ret i32 %sel +} + +; Should NOT be folded: icmp eq with different constants +define i32 @icmp_select_no_fold_i32_different(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_no_fold_i32_different: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x978 +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i32_different: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x978, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i32 %arg, 4242 + %sel = select i1 %cmp, i32 2424, i32 %other + ret i32 %sel +} + +; Should NOT be folded: icmp eq with constant in other position +define i32 @icmp_select_no_fold_i32_other_pos(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_no_fold_i32_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x1092 +; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i32_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x1092, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i32 %arg, 4242 + %sel = select i1 %cmp, i32 %other, i32 4242 + ret i32 %sel +} + +; Should NOT be folded: unsupported comparison type +define i32 @icmp_select_no_fold_i32_unsupported_cmp(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_no_fold_i32_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1094 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x102d +; GFX900-NEXT: v_cmp_gt_u32_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i32_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_gt_u32_e32 vcc_lo, 0x1094, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x102d, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ugt i32 %arg, 4243 + %sel = select i1 %cmp, i32 4141, i32 %other + ret i32 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i32 @icmp_select_no_fold_i32_enc_imm(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_no_fold_i32_enc_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i32_enc_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i32 %arg, 0 + %sel = select i1 %cmp, i32 0, i32 %other + ret i32 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i32 @icmp_select_no_fold_i32_enc_imm_2(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_no_fold_i32_enc_imm_2: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, 64, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 64, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i32_enc_imm_2: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u32_e32 vcc_lo, 64, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 64, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i32 64, %arg + %sel = select i1 %cmp, i32 64, i32 %other + ret i32 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i32 @icmp_select_no_fold_i32_enc_imm_3(i32 %arg, i32 %other) { +; GFX900-LABEL: icmp_select_no_fold_i32_enc_imm_3: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, -16, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, -16, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i32_enc_imm_3: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u32_e32 vcc_lo, -16, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, -16, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i32 %arg, -16 + %sel = select i1 %cmp, i32 %other, i32 -16 + ret i32 %sel +} + +;------------------------------------------------------------------------------ +; I64 Tests +;------------------------------------------------------------------------------ + +; Should be folded: icmp eq + select with constant in true value +define i64 @icmp_select_fold_eq_i64_imm(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_fold_eq_i64_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_i64_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i64 %arg, 424242424242 + %sel = select i1 %cmp, i64 424242424242, i64 %other + ret i64 %sel +} + +; Should be folded: icmp eq + select with constant in true value (commutative) +define i64 @icmp_select_fold_eq_imm_i64(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_fold_eq_imm_i64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_imm_i64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i64 424242424242, %arg + %sel = select i1 %cmp, i64 424242424242, i64 %other + ret i64 %sel +} + +; Should be folded: icmp ne + select with constant in false value +define i64 @icmp_select_fold_ne_i64_imm(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_fold_ne_i64_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_ne_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_i64_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_ne_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i64 %arg, 424242424242 + %sel = select i1 %cmp, i64 %other, i64 424242424242 + ret i64 %sel +} + +; Should be folded: icmp ne + select with constant in false value (commutative) +define i64 @icmp_select_fold_ne_imm_i64(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_fold_ne_imm_i64: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_ne_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_imm_i64: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_ne_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i64 424242424242, %arg + %sel = select i1 %cmp, i64 %other, i64 424242424242 + ret i64 %sel +} + +; Should NOT be folded: icmp eq with different constants +define i64 @icmp_select_no_fold_i64_different(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_no_fold_i64_different: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_ne_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v4, 0x719c60f8 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, 56, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i64_different: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_ne_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x719c60f8, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 56, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i64 %arg, 424242424242 + %sel = select i1 %cmp, i64 242424242424, i64 %other + ret i64 %sel +} + +; Should NOT be folded: icmp eq with constant in other position +define i64 @icmp_select_no_fold_i64_other_pos(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_no_fold_i64_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v4, 0xc6d1a9b2 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x62 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i64_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b2 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0xc6d1a9b2, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x62, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i64 %arg, 424242424242 + %sel = select i1 %cmp, i64 %other, i64 424242424242 + ret i64 %sel +} + +; Should NOT be folded: unsupported comparison type +define i64 @icmp_select_no_fold_i64_unsupported_cmp(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_no_fold_i64_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_mov_b32 s4, 0xc6d1a9b3 +; GFX900-NEXT: s_movk_i32 s5, 0x62 +; GFX900-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1] +; GFX900-NEXT: v_mov_b32_e32 v4, 0xc6d1a9b2 +; GFX900-NEXT: v_mov_b32_e32 v1, 0x62 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i64_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: s_mov_b32 s4, 0xc6d1a9b3 +; GFX1010-NEXT: s_movk_i32 s5, 0x62 +; GFX1010-NEXT: v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0xc6d1a9b2, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0x62, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ugt i64 %arg, 424242424242 + %sel = select i1 %cmp, i64 424242424242, i64 %other + ret i64 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i64 @icmp_select_no_fold_i64_enc_imm(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_no_fold_i64_enc_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i64_enc_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i64 %arg, 0 + %sel = select i1 %cmp, i64 0, i64 %other + ret i64 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i64 @icmp_select_no_fold_i64_enc_imm_2(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_no_fold_i64_enc_imm_2: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u64_e32 vcc, 32, v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, 32, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i64_enc_imm_2: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u64_e32 vcc_lo, 32, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 32, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i64 32, %arg + %sel = select i1 %cmp, i64 32, i64 %other + ret i64 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i64 @icmp_select_no_fold_i64_enc_imm_3(i64 %arg, i64 %other) { +; GFX900-LABEL: icmp_select_no_fold_i64_enc_imm_3: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u64_e32 vcc, -8, v[0:1] +; GFX900-NEXT: v_cndmask_b32_e32 v0, -8, v2, vcc +; GFX900-NEXT: v_cndmask_b32_e32 v1, -1, v3, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i64_enc_imm_3: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u64_e32 vcc_lo, -8, v[0:1] +; GFX1010-NEXT: v_cndmask_b32_e32 v0, -8, v2, vcc_lo +; GFX1010-NEXT: v_cndmask_b32_e32 v1, -1, v3, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i64 %arg, -8 + %sel = select i1 %cmp, i64 %other, i64 -8 + ret i64 %sel +} + +;------------------------------------------------------------------------------ +; I16 Tests +;------------------------------------------------------------------------------ + +; Should be folded: icmp eq + select with constant in true value +define i16 @icmp_select_fold_eq_i16_imm(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_fold_eq_i16_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_eq_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_i16_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i16 %arg, 4242 + %sel = select i1 %cmp, i16 4242, i16 %other + ret i16 %sel +} + +; Should be folded: icmp eq + select with constant in true value (commutative) +define i16 @icmp_select_fold_eq_imm_i16(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_fold_eq_imm_i16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_eq_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_imm_i16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i16 4242, %arg + %sel = select i1 %cmp, i16 4242, i16 %other + ret i16 %sel +} + +; Should be folded: icmp ne + select with constant in false value +define i16 @icmp_select_fold_ne_i16_imm(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_fold_ne_i16_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_ne_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_i16_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i16 %arg, 4242 + %sel = select i1 %cmp, i16 %other, i16 4242 + ret i16 %sel +} + +; Should be folded: icmp ne + select with constant in false value (commutative) +define i16 @icmp_select_fold_ne_imm_i16(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_fold_ne_imm_i16: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_cmp_ne_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_imm_i16: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i16 4242, %arg + %sel = select i1 %cmp, i16 %other, i16 4242 + ret i16 %sel +} + +; Should NOT be folded: icmp eq with different constants +define i16 @icmp_select_no_fold_i16_different(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_no_fold_i16_different: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x978 +; GFX900-NEXT: v_cmp_ne_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i16_different: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x978, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i16 %arg, 4242 + %sel = select i1 %cmp, i16 2424, i16 %other + ret i16 %sel +} + +; Should NOT be folded: icmp eq with constant in other position +define i16 @icmp_select_no_fold_i16_other_pos(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_no_fold_i16_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1092 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x1092 +; GFX900-NEXT: v_cmp_eq_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i16_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x1092, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x1092, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i16 %arg, 4242 + %sel = select i1 %cmp, i16 %other, i16 4242 + ret i16 %sel +} + +; Should NOT be folded: unsupported comparison type +define i16 @icmp_select_no_fold_i16_unsupported_cmp(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_no_fold_i16_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x1093 +; GFX900-NEXT: v_mov_b32_e32 v2, 0x1092 +; GFX900-NEXT: v_cmp_gt_u16_e32 vcc, s4, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i16_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_gt_u16_e32 vcc_lo, 0x1093, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x1092, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ugt i16 %arg, 4242 + %sel = select i1 %cmp, i16 4242, i16 %other + ret i16 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i16 @icmp_select_no_fold_i16_enc_imm(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_no_fold_i16_enc_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i16_enc_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i16 %arg, 0 + %sel = select i1 %cmp, i16 0, i16 %other + ret i16 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i16 @icmp_select_no_fold_i16_enc_imm_2(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_no_fold_i16_enc_imm_2: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u16_e32 vcc, 45, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, 45, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i16_enc_imm_2: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u16_e32 vcc_lo, 45, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 45, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i16 45, %arg + %sel = select i1 %cmp, i16 45, i16 %other + ret i16 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i16 @icmp_select_no_fold_i16_enc_imm_3(i16 %arg, i16 %other) { +; GFX900-LABEL: icmp_select_no_fold_i16_enc_imm_3: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_cmp_ne_u16_e32 vcc, -12, v0 +; GFX900-NEXT: v_cndmask_b32_e32 v0, -12, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i16_enc_imm_3: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_cmp_ne_u16_e32 vcc_lo, -12, v0 +; GFX1010-NEXT: v_cndmask_b32_e32 v0, -12, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i16 %arg, -12 + %sel = select i1 %cmp, i16 %other, i16 -12 + ret i16 %sel +} + +;------------------------------------------------------------------------------ +; I8 Tests +;------------------------------------------------------------------------------ + +; Should be folded: icmp eq + select with constant in true value +define i8 @icmp_select_fold_eq_i8_imm(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_fold_eq_i8_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7b +; GFX900-NEXT: v_cmp_eq_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_i8_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX1010-NEXT: v_cmp_eq_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i8 %arg, 123 + %sel = select i1 %cmp, i8 123, i8 %other + ret i8 %sel +} + +; Should be folded: icmp eq + select with constant in true value (commutative) +define i8 @icmp_select_fold_eq_imm_i8(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_fold_eq_imm_i8: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7b +; GFX900-NEXT: v_cmp_eq_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_eq_imm_i8: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX1010-NEXT: v_cmp_eq_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i8 123, %arg + %sel = select i1 %cmp, i8 123, i8 %other + ret i8 %sel +} + +; Should be folded: icmp ne + select with constant in false value +define i8 @icmp_select_fold_ne_i8_imm(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_fold_ne_i8_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7b +; GFX900-NEXT: v_cmp_ne_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_i8_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX1010-NEXT: v_cmp_ne_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i8 %arg, 123 + %sel = select i1 %cmp, i8 %other, i8 123 + ret i8 %sel +} + +; Should be folded: icmp ne + select with constant in false value (commutative) +define i8 @icmp_select_fold_ne_imm_i8(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_fold_ne_imm_i8: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7b +; GFX900-NEXT: v_cmp_ne_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_fold_ne_imm_i8: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX1010-NEXT: v_cmp_ne_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i8 123, %arg + %sel = select i1 %cmp, i8 %other, i8 123 + ret i8 %sel +} + +; Should NOT be folded: icmp eq with different constants +define i8 @icmp_select_no_fold_i8_different(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_no_fold_i8_different: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7b +; GFX900-NEXT: v_mov_b32_e32 v2, 0x7c +; GFX900-NEXT: v_cmp_ne_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i8_different: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX1010-NEXT: v_cmp_ne_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x7c, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i8 %arg, 123 + %sel = select i1 %cmp, i8 124, i8 %other + ret i8 %sel +} + +; Should NOT be folded: icmp eq with constant in other position +define i8 @icmp_select_no_fold_i8_other_pos(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_no_fold_i8_other_pos: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7b +; GFX900-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX900-NEXT: v_cmp_eq_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i8_other_pos: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX1010-NEXT: v_cmp_eq_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x7b, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i8 %arg, 123 + %sel = select i1 %cmp, i8 %other, i8 123 + ret i8 %sel +} + +; Should NOT be folded: unsupported comparison type +define i8 @icmp_select_no_fold_i8_unsupported_cmp(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_no_fold_i8_unsupported_cmp: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0x7c +; GFX900-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX900-NEXT: v_cmp_lt_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i8_unsupported_cmp: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0x7c +; GFX1010-NEXT: v_cmp_lt_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0x7b, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ugt i8 %arg, 123 + %sel = select i1 %cmp, i8 123, i8 %other + ret i8 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i8 @icmp_select_no_fold_i8_enc_imm(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_no_fold_i8_enc_imm: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v2, 0 +; GFX900-NEXT: v_cmp_ne_u16_sdwa vcc, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i8_enc_imm: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0 +; GFX1010-NEXT: v_cmp_ne_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i8 %arg, 0 + %sel = select i1 %cmp, i8 0, i8 %other + ret i8 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i8 @icmp_select_no_fold_i8_enc_imm_2(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_no_fold_i8_enc_imm_2: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: v_mov_b32_e32 v2, 25 +; GFX900-NEXT: v_cmp_ne_u16_sdwa vcc, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, 25, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i8_enc_imm_2: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 25 +; GFX1010-NEXT: v_cmp_ne_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, 25, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp eq i8 25, %arg + %sel = select i1 %cmp, i8 25, i8 %other + ret i8 %sel +} + +; Should NOT be folded: immediate can be encoded into cndmask +define i8 @icmp_select_no_fold_i8_enc_imm_3(i8 %arg, i8 %other) { +; GFX900-LABEL: icmp_select_no_fold_i8_enc_imm_3: +; GFX900: ; %bb.0: ; %entry +; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-NEXT: s_movk_i32 s4, 0xfb +; GFX900-NEXT: v_cmp_ne_u16_sdwa vcc, v0, s4 src0_sel:BYTE_0 src1_sel:DWORD +; GFX900-NEXT: v_cndmask_b32_e32 v0, -5, v1, vcc +; GFX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX1010-LABEL: icmp_select_no_fold_i8_enc_imm_3: +; GFX1010: ; %bb.0: ; %entry +; GFX1010-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX1010-NEXT: v_mov_b32_e32 v2, 0xfb +; GFX1010-NEXT: v_cmp_ne_u16_sdwa vcc_lo, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD +; GFX1010-NEXT: v_cndmask_b32_e32 v0, -5, v1, vcc_lo +; GFX1010-NEXT: s_setpc_b64 s[30:31] +entry: + %cmp = icmp ne i8 %arg, -5 + %sel = select i1 %cmp, i8 %other, i8 -5 + ret i8 %sel +} diff --git a/llvm/test/CodeGen/AMDGPU/select-undef.ll b/llvm/test/CodeGen/AMDGPU/select-undef.ll index ec3781f..f497752 100644 --- a/llvm/test/CodeGen/AMDGPU/select-undef.ll +++ b/llvm/test/CodeGen/AMDGPU/select-undef.ll @@ -841,3 +841,23 @@ ret: ret void } +define i64 @poison_should_freeze(i1 %cond1, i32 %val, i16 %val2, i64 %a, i64 %b) { +; GCN-LABEL: poison_should_freeze: +; GCN: ; %bb.0: +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: v_and_b32_e32 v0, 1, v0 +; GCN-NEXT: v_mov_b32_e32 v7, 0x5040100 +; GCN-NEXT: v_perm_b32 v2, v2, s4, v7 +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v5, v3, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc +; GCN-NEXT: s_setpc_b64 s[30:31] + %poisonv = insertelement <2 x i16> poison, i16 %val2, i32 1 + %poison = bitcast <2 x i16> %poisonv to i32 + %cond2 = select i1 %cond1, i32 %poison, i32 %val + %cmp = icmp eq i32 %cond2, 0 + %select = select i1 %cmp, i64 %a, i64 %b + ret i64 %select +} diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll new file mode 100644 index 0000000..192bd20 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll @@ -0,0 +1,100 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefixes=GCN %s + +define amdgpu_kernel void @copy_to_vreg_1(i32 %0) { +; GCN-LABEL: copy_to_vreg_1: +; GCN: ; %bb.0: ; %._crit_edge +; GCN-NEXT: s_load_dword s4, s[4:5], 0x0 +; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GCN-NEXT: v_mov_b64_e32 v[2:3], 0 +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_sub_i32 s5, 1, s4 +; GCN-NEXT: s_cmp_lt_u32 s4, 2 +; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0 +; GCN-NEXT: s_and_b64 s[2:3], s[0:1], exec +; GCN-NEXT: s_cselect_b32 s3, s5, 1 +; GCN-NEXT: s_cmp_lg_u64 s[0:1], 0 +; GCN-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1] +; GCN-NEXT: s_addc_u32 s0, 1, 0 +; GCN-NEXT: v_readfirstlane_b32 s2, v1 +; GCN-NEXT: s_cmp_ge_u32 s3, s4 +; GCN-NEXT: s_cselect_b32 s4, s0, s2 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: s_cmp_lg_u64 0, 0 +; GCN-NEXT: s_mov_b64 s[0:1], 0 +; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] +; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0 +; GCN-NEXT: s_branch .LBB0_3 +; GCN-NEXT: .LBB0_1: ; %Flow +; GCN-NEXT: ; in Loop: Header=BB0_3 Depth=1 +; GCN-NEXT: s_or_b64 exec, exec, s[6:7] +; GCN-NEXT: s_xor_b64 s[8:9], exec, -1 +; GCN-NEXT: .LBB0_2: ; %Flow3 +; GCN-NEXT: ; in Loop: Header=BB0_3 Depth=1 +; GCN-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-NEXT: s_and_b64 s[4:5], exec, s[8:9] +; GCN-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1] +; GCN-NEXT: s_mov_b32 s4, 0 +; GCN-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GCN-NEXT: s_cbranch_execz .LBB0_8 +; GCN-NEXT: .LBB0_3: ; %.lr.ph27 +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: s_cmp_lg_u32 s4, 0 +; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0 +; GCN-NEXT: s_or_b64 s[8:9], vcc, s[4:5] +; GCN-NEXT: s_xor_b64 s[6:7], s[8:9], -1 +; GCN-NEXT: s_and_saveexec_b64 s[4:5], s[8:9] +; GCN-NEXT: s_cbranch_execz .LBB0_5 +; GCN-NEXT: ; %bb.4: ; %pred.store.if +; GCN-NEXT: ; in Loop: Header=BB0_3 Depth=1 +; GCN-NEXT: s_or_b64 s[6:7], s[6:7], exec +; GCN-NEXT: global_store_byte v[2:3], v1, off +; GCN-NEXT: .LBB0_5: ; %Flow2 +; GCN-NEXT: ; in Loop: Header=BB0_3 Depth=1 +; GCN-NEXT: s_or_b64 exec, exec, s[4:5] +; GCN-NEXT: s_mov_b64 s[8:9], -1 +; GCN-NEXT: s_and_saveexec_b64 s[4:5], s[6:7] +; GCN-NEXT: s_cbranch_execz .LBB0_2 +; GCN-NEXT: ; %bb.6: ; %pred.store.continue +; GCN-NEXT: ; in Loop: Header=BB0_3 Depth=1 +; GCN-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] +; GCN-NEXT: s_xor_b64 s[6:7], exec, s[6:7] +; GCN-NEXT: s_cbranch_execz .LBB0_1 +; GCN-NEXT: ; %bb.7: ; %pred.store.if41 +; GCN-NEXT: ; in Loop: Header=BB0_3 Depth=1 +; GCN-NEXT: global_store_byte v[2:3], v1, off +; GCN-NEXT: s_branch .LBB0_1 +; GCN-NEXT: .LBB0_8: ; %DummyReturnBlock +; GCN-NEXT: s_endpgm +._crit_edge: + %id.x = tail call i32 @llvm.amdgcn.workitem.id.x() + %div = udiv i32 1, %0 + br label %.lr.ph27 + +.lr.ph27: ; preds = %pred.store.if41, %pred.store.continue, %._crit_edge + %iv = phi i32 [ %div, %._crit_edge ], [ 0, %pred.store.if41 ], [ 0, %pred.store.continue ] + %cmp = icmp ugt i32 %iv, 0 + %broadcast.splatinsert37 = insertelement <4 x i1> zeroinitializer, i1 %cmp, i64 0 + %.zext = zext i32 %id.x to i64 + %broadcast.splatinsert39 = insertelement <4 x i64> zeroinitializer, i64 %.zext, i64 0 + %cmp.1 = icmp uge <4 x i64> %broadcast.splatinsert39, splat (i64 1) + %or = or <4 x i1> %cmp.1, %broadcast.splatinsert37 + %extract = extractelement <4 x i1> %or, i64 0 + br i1 %extract, label %pred.store.if, label %pred.store.continue + +pred.store.if: ; preds = %.lr.ph27 + store i8 0, ptr addrspace(1) null, align 64 + br label %pred.store.continue + +pred.store.continue: ; preds = %pred.store.if, %.lr.ph27 + %extract.1 = extractelement <4 x i1> %or, i64 1 + br i1 %extract.1, label %pred.store.if41, label %.lr.ph27 + +pred.store.if41: ; preds = %pred.store.continue + store i8 0, ptr addrspace(1) null, align 64 + br label %.lr.ph27 +} + +declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #0 + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.mir b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.mir new file mode 100644 index 0000000..2daea2b --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.mir @@ -0,0 +1,31 @@ +# RUN: llc -mtriple=amdgcn -run-pass si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s + +--- +name: copy_to_vreg_1 +tracksRegLiveness: true +body: | + ; GCN-LABEL: name: copy_to_vreg_1 + ; GCN: bb.0: + ; GCN-NEXT: successors: %bb.1(0x80000000) + ; GCN-NEXT: liveins: $vgpr0, $vgpr1 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[V_CVT_U32_F32_e64:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_U32_F32_e64 0, killed $vgpr0, 0, 0, implicit $mode, implicit $exec + ; GCN-NEXT: [[IMPLICIT_DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF + ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1 + ; GCN-NEXT: [[V_CMP_GT_U32_e64:%[0-9]+]]:sreg_64_xexec = samesign V_CMP_GT_U32_e64 [[V_CVT_U32_F32_e64]], killed [[COPY1]], implicit $exec + ; GCN-NEXT: [[VREG1:%[0-9]+]]:vreg_1 = COPY [[V_CMP_GT_U32_e64]] + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: bb.1: + ; GCN-NEXT: S_ENDPGM 0 + bb.0: + liveins: $vgpr0, $vgpr1 + %0:vgpr_32 = nofpexcept V_CVT_U32_F32_e64 0, killed $vgpr0, 0, 0, implicit $mode, implicit $exec + %1:sreg_32 = COPY %0:vgpr_32 + %2:sreg_32 = COPY $vgpr1 + samesign S_CMP_GT_U32 %1:sreg_32, killed %2:sreg_32, implicit-def $scc + %3:sreg_64 = COPY $scc + %4:vreg_1 = COPY %3:sreg_64 + + bb.1: + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.mir b/llvm/test/CodeGen/AMDGPU/spill-agpr.mir index 8e6da4bf9..3f6956b 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-agpr.mir +++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.mir @@ -18,9 +18,9 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0 - ; GFX908-SPILLED-NEXT: SI_SPILL_A32_SAVE killed $agpr0, %stack.1, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV32_SAVE killed $agpr0, %stack.1, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0 - ; GFX908-SPILLED-NEXT: SI_SPILL_A32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -29,8 +29,8 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0 = SI_SPILL_A32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5) - ; GFX908-SPILLED-NEXT: $agpr1 = SI_SPILL_A32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0 = SI_SPILL_AV32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr1 = SI_SPILL_AV32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0, implicit killed renamable $agpr1 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr32 @@ -62,9 +62,9 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A32_SAVE killed $agpr0, %stack.1, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV32_SAVE killed $agpr0, %stack.1, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -73,8 +73,8 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0 = SI_SPILL_A32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5) - ; GFX90A-SPILLED-NEXT: $agpr1 = SI_SPILL_A32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0 = SI_SPILL_AV32_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.1, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr1 = SI_SPILL_AV32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0, implicit killed renamable $agpr1 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr32 @@ -124,7 +124,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1 - ; GFX908-SPILLED-NEXT: SI_SPILL_A64_SAVE killed $agpr0_agpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV64_SAVE killed $agpr0_agpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -133,7 +133,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1 = SI_SPILL_A64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr64 @@ -164,7 +164,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A64_SAVE killed $agpr0_agpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV64_SAVE killed $agpr0_agpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -173,7 +173,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1 = SI_SPILL_A64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr64 @@ -222,14 +222,14 @@ body: | ; GFX908-SPILLED-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251_vgpr252_vgpr253_vgpr254_vgpr255 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0 - ; GFX908-SPILLED-NEXT: SI_SPILL_A32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: ; GFX908-SPILLED-NEXT: successors: %bb.2(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0 = SI_SPILL_A32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0 = SI_SPILL_AV32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; GFX908-SPILLED-NEXT: S_NOP 0, implicit undef $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 ; GFX908-SPILLED-NEXT: S_NOP 0, implicit undef $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 @@ -288,14 +288,14 @@ body: | ; GFX90A-SPILLED-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47, $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63, $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79, $vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95, $vgpr96_vgpr97_vgpr98_vgpr99_vgpr100_vgpr101_vgpr102_vgpr103_vgpr104_vgpr105_vgpr106_vgpr107_vgpr108_vgpr109_vgpr110_vgpr111, $vgpr112_vgpr113_vgpr114_vgpr115_vgpr116_vgpr117_vgpr118_vgpr119_vgpr120_vgpr121_vgpr122_vgpr123_vgpr124_vgpr125_vgpr126_vgpr127, $vgpr128_vgpr129_vgpr130_vgpr131_vgpr132_vgpr133_vgpr134_vgpr135_vgpr136_vgpr137_vgpr138_vgpr139_vgpr140_vgpr141_vgpr142_vgpr143, $vgpr144_vgpr145_vgpr146_vgpr147_vgpr148_vgpr149_vgpr150_vgpr151_vgpr152_vgpr153_vgpr154_vgpr155_vgpr156_vgpr157_vgpr158_vgpr159, $vgpr160_vgpr161_vgpr162_vgpr163_vgpr164_vgpr165_vgpr166_vgpr167_vgpr168_vgpr169_vgpr170_vgpr171_vgpr172_vgpr173_vgpr174_vgpr175, $vgpr176_vgpr177_vgpr178_vgpr179_vgpr180_vgpr181_vgpr182_vgpr183_vgpr184_vgpr185_vgpr186_vgpr187_vgpr188_vgpr189_vgpr190_vgpr191, $vgpr192_vgpr193_vgpr194_vgpr195_vgpr196_vgpr197_vgpr198_vgpr199_vgpr200_vgpr201_vgpr202_vgpr203_vgpr204_vgpr205_vgpr206_vgpr207, $vgpr208_vgpr209_vgpr210_vgpr211_vgpr212_vgpr213_vgpr214_vgpr215_vgpr216_vgpr217_vgpr218_vgpr219_vgpr220_vgpr221_vgpr222_vgpr223, $vgpr224_vgpr225_vgpr226_vgpr227_vgpr228_vgpr229_vgpr230_vgpr231_vgpr232_vgpr233_vgpr234_vgpr235_vgpr236_vgpr237_vgpr238_vgpr239, $vgpr240_vgpr241_vgpr242_vgpr243_vgpr244_vgpr245_vgpr246_vgpr247, $vgpr248_vgpr249_vgpr250_vgpr251_vgpr252_vgpr253_vgpr254_vgpr255 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.0, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: ; GFX90A-SPILLED-NEXT: successors: %bb.2(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0 = SI_SPILL_A32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0 = SI_SPILL_AV32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.0, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit undef $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47 ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit undef $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63 @@ -385,7 +385,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2 - ; GFX908-SPILLED-NEXT: SI_SPILL_A96_SAVE killed $agpr0_agpr1_agpr2, %stack.0, $sgpr32, 0, implicit $exec :: (store (s96) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV96_SAVE killed $agpr0_agpr1_agpr2, %stack.0, $sgpr32, 0, implicit $exec :: (store (s96) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -394,7 +394,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2 = SI_SPILL_A96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2 = SI_SPILL_AV96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr96 @@ -427,7 +427,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A96_SAVE killed $agpr0_agpr1_agpr2, %stack.0, $sgpr32, 0, implicit $exec :: (store (s96) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV96_SAVE killed $agpr0_agpr1_agpr2, %stack.0, $sgpr32, 0, implicit $exec :: (store (s96) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -436,7 +436,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2 = SI_SPILL_A96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2 = SI_SPILL_AV96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s96) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr96 @@ -486,7 +486,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3 - ; GFX908-SPILLED-NEXT: SI_SPILL_A128_SAVE killed $agpr0_agpr1_agpr2_agpr3, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV128_SAVE killed $agpr0_agpr1_agpr2_agpr3, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -495,7 +495,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3 = SI_SPILL_A128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3 = SI_SPILL_AV128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr128 @@ -530,7 +530,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A128_SAVE killed $agpr0_agpr1_agpr2_agpr3, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV128_SAVE killed $agpr0_agpr1_agpr2_agpr3, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -539,7 +539,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3 = SI_SPILL_A128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3 = SI_SPILL_AV128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr128 @@ -591,7 +591,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4 - ; GFX908-SPILLED-NEXT: SI_SPILL_A160_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s160) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV160_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s160) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -600,7 +600,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4 = SI_SPILL_A160_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s160) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4 = SI_SPILL_AV160_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s160) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr160 @@ -637,7 +637,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A160_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s160) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV160_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s160) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -646,7 +646,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4 = SI_SPILL_A160_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s160) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4 = SI_SPILL_AV160_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s160) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr160 @@ -700,7 +700,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 - ; GFX908-SPILLED-NEXT: SI_SPILL_A192_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store (s192) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV192_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store (s192) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -709,7 +709,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = SI_SPILL_A192_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s192) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = SI_SPILL_AV192_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s192) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr192 @@ -748,7 +748,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A192_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store (s192) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV192_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store (s192) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -757,7 +757,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = SI_SPILL_A192_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s192) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = SI_SPILL_AV192_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s192) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr192 @@ -813,7 +813,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 - ; GFX908-SPILLED-NEXT: SI_SPILL_A256_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, %stack.0, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV256_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, %stack.0, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -822,7 +822,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = SI_SPILL_A256_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = SI_SPILL_AV256_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr256 @@ -865,7 +865,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A256_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, %stack.0, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV256_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, %stack.0, $sgpr32, 0, implicit $exec :: (store (s256) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -874,7 +874,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = SI_SPILL_A256_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = SI_SPILL_AV256_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s256) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr256 @@ -934,7 +934,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 - ; GFX908-SPILLED-NEXT: SI_SPILL_A288_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8, %stack.0, $sgpr32, 0, implicit $exec :: (store (s288) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV288_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8, %stack.0, $sgpr32, 0, implicit $exec :: (store (s288) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -943,7 +943,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 = SI_SPILL_A288_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s288) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 = SI_SPILL_AV288_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s288) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr288 @@ -988,7 +988,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A288_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8, %stack.0, $sgpr32, 0, implicit $exec :: (store (s288) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV288_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8, %stack.0, $sgpr32, 0, implicit $exec :: (store (s288) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -997,7 +997,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 = SI_SPILL_A288_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s288) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 = SI_SPILL_AV288_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s288) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr288 @@ -1059,7 +1059,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 - ; GFX908-SPILLED-NEXT: SI_SPILL_A320_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9, %stack.0, $sgpr32, 0, implicit $exec :: (store (s320) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV320_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9, %stack.0, $sgpr32, 0, implicit $exec :: (store (s320) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -1068,7 +1068,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 = SI_SPILL_A320_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s320) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 = SI_SPILL_AV320_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s320) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr320 @@ -1115,7 +1115,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A320_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9, %stack.0, $sgpr32, 0, implicit $exec :: (store (s320) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV320_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9, %stack.0, $sgpr32, 0, implicit $exec :: (store (s320) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -1124,7 +1124,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 = SI_SPILL_A320_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s320) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 = SI_SPILL_AV320_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s320) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr320 @@ -1188,7 +1188,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 - ; GFX908-SPILLED-NEXT: SI_SPILL_A352_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10, %stack.0, $sgpr32, 0, implicit $exec :: (store (s352) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV352_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10, %stack.0, $sgpr32, 0, implicit $exec :: (store (s352) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -1197,7 +1197,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 = SI_SPILL_A352_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s352) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 = SI_SPILL_AV352_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s352) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr352 @@ -1246,7 +1246,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A352_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10, %stack.0, $sgpr32, 0, implicit $exec :: (store (s352) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV352_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10, %stack.0, $sgpr32, 0, implicit $exec :: (store (s352) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -1255,7 +1255,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 = SI_SPILL_A352_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s352) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 = SI_SPILL_AV352_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s352) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr352 @@ -1321,7 +1321,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 - ; GFX908-SPILLED-NEXT: SI_SPILL_A384_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11, %stack.0, $sgpr32, 0, implicit $exec :: (store (s384) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV384_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11, %stack.0, $sgpr32, 0, implicit $exec :: (store (s384) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -1330,7 +1330,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 = SI_SPILL_A384_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s384) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 = SI_SPILL_AV384_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s384) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr384 @@ -1381,7 +1381,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A384_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11, %stack.0, $sgpr32, 0, implicit $exec :: (store (s384) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV384_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11, %stack.0, $sgpr32, 0, implicit $exec :: (store (s384) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -1390,7 +1390,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 = SI_SPILL_A384_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s384) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 = SI_SPILL_AV384_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s384) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr384 @@ -1458,7 +1458,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; GFX908-SPILLED-NEXT: SI_SPILL_A512_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV512_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -1467,7 +1467,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = SI_SPILL_A512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr512 @@ -1526,7 +1526,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A512_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV512_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -1535,7 +1535,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = SI_SPILL_A512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr512 @@ -1611,7 +1611,7 @@ body: | ; GFX908-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 - ; GFX908-SPILLED-NEXT: SI_SPILL_A1024_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, %stack.0, $sgpr32, 0, implicit $exec :: (store (s1024) into %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: SI_SPILL_AV1024_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, %stack.0, $sgpr32, 0, implicit $exec :: (store (s1024) into %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.1: @@ -1620,7 +1620,7 @@ body: | ; GFX908-SPILLED-NEXT: S_NOP 1 ; GFX908-SPILLED-NEXT: {{ $}} ; GFX908-SPILLED-NEXT: bb.2: - ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = SI_SPILL_A1024_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s1024) from %stack.0, align 4, addrspace 5) + ; GFX908-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = SI_SPILL_AV1024_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s1024) from %stack.0, align 4, addrspace 5) ; GFX908-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 ; ; GFX908-EXPANDED-LABEL: name: spill_restore_agpr1024 @@ -1711,7 +1711,7 @@ body: | ; GFX90A-SPILLED-NEXT: successors: %bb.1(0x80000000) ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 - ; GFX90A-SPILLED-NEXT: SI_SPILL_A1024_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, %stack.0, $sgpr32, 0, implicit $exec :: (store (s1024) into %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: SI_SPILL_AV1024_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, %stack.0, $sgpr32, 0, implicit $exec :: (store (s1024) into %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.1: @@ -1720,7 +1720,7 @@ body: | ; GFX90A-SPILLED-NEXT: S_NOP 1 ; GFX90A-SPILLED-NEXT: {{ $}} ; GFX90A-SPILLED-NEXT: bb.2: - ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = SI_SPILL_A1024_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s1024) from %stack.0, align 4, addrspace 5) + ; GFX90A-SPILLED-NEXT: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = SI_SPILL_AV1024_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s1024) from %stack.0, align 4, addrspace 5) ; GFX90A-SPILLED-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 ; ; GFX90A-EXPANDED-LABEL: name: spill_restore_agpr1024 diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll index bd255e8..648b59f 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll +++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll @@ -9,9 +9,9 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 { ; GCN-NEXT: {{ $}} ; GCN-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4) ; GCN-NEXT: [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3 - ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec - ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec - ; GCN-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec + ; GCN-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec + ; GCN-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec + ; GCN-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec ; GCN-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %14.sub0 ; GCN-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]] ; GCN-NEXT: GLOBAL_STORE_DWORDX4 undef %24:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1) diff --git a/llvm/test/CodeGen/AMDGPU/srem.ll b/llvm/test/CodeGen/AMDGPU/srem.ll index 6da7d1b..a6b8ea3 100644 --- a/llvm/test/CodeGen/AMDGPU/srem.ll +++ b/llvm/test/CodeGen/AMDGPU/srem.ll @@ -1819,7 +1819,7 @@ define amdgpu_kernel void @srem_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) ; TAHITI-NEXT: v_mul_hi_u32 v1, v0, v1 ; TAHITI-NEXT: v_mul_lo_u32 v1, v1, v2 ; TAHITI-NEXT: v_sub_i32_e32 v0, vcc, v0, v1 -; TAHITI-NEXT: v_subrev_i32_e32 v1, vcc, v2, v0 +; TAHITI-NEXT: v_sub_i32_e32 v1, vcc, v0, v2 ; TAHITI-NEXT: v_cmp_ge_u32_e32 vcc, v0, v2 ; TAHITI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; TAHITI-NEXT: v_sub_i32_e32 v1, vcc, v0, v2 @@ -6232,7 +6232,7 @@ define amdgpu_kernel void @srem_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %i ; TONGA-NEXT: v_mul_hi_u32 v8, v14, v8 ; TONGA-NEXT: v_mul_lo_u32 v8, v8, v10 ; TONGA-NEXT: v_sub_u32_e32 v8, vcc, v14, v8 -; TONGA-NEXT: v_subrev_u32_e32 v9, vcc, v10, v8 +; TONGA-NEXT: v_sub_u32_e32 v9, vcc, v8, v10 ; TONGA-NEXT: v_cmp_ge_u32_e32 vcc, v8, v10 ; TONGA-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc ; TONGA-NEXT: v_sub_u32_e32 v9, vcc, v8, v10 diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll index 47dfa9f..33c2ce6 100644 --- a/llvm/test/CodeGen/AMDGPU/srem64.ll +++ b/llvm/test/CodeGen/AMDGPU/srem64.ll @@ -921,45 +921,47 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 ; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i64 s[10:11], s[2:3], 31 -; GCN-NEXT: s_ashr_i64 s[6:7], s[4:5], 31 -; GCN-NEXT: s_ashr_i32 s4, s5, 31 -; GCN-NEXT: s_add_u32 s6, s6, s4 -; GCN-NEXT: s_mov_b32 s5, s4 -; GCN-NEXT: s_addc_u32 s7, s7, s4 -; GCN-NEXT: s_xor_b64 s[8:9], s[6:7], s[4:5] +; GCN-NEXT: s_ashr_i64 s[2:3], s[2:3], 31 +; GCN-NEXT: s_ashr_i64 s[4:5], s[4:5], 31 +; GCN-NEXT: s_ashr_i32 s6, s5, 31 +; GCN-NEXT: s_add_u32 s4, s4, s6 +; GCN-NEXT: s_mov_b32 s7, s6 +; GCN-NEXT: s_addc_u32 s5, s5, s6 +; GCN-NEXT: s_xor_b64 s[8:9], s[4:5], s[6:7] ; GCN-NEXT: v_cvt_f32_u32_e32 v0, s8 ; GCN-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GCN-NEXT: s_sub_u32 s2, 0, s8 -; GCN-NEXT: s_subb_u32 s4, 0, s9 -; GCN-NEXT: s_ashr_i32 s12, s3, 31 +; GCN-NEXT: s_sub_u32 s4, 0, s8 +; GCN-NEXT: s_subb_u32 s5, 0, s9 +; GCN-NEXT: s_ashr_i32 s10, s3, 31 ; GCN-NEXT: v_madmk_f32 v0, v1, 0x4f800000, v0 ; GCN-NEXT: v_rcp_f32_e32 v0, v0 -; GCN-NEXT: s_mov_b32 s13, s12 -; GCN-NEXT: s_mov_b32 s5, s1 -; GCN-NEXT: s_mov_b32 s7, 0xf000 +; GCN-NEXT: s_add_u32 s2, s2, s10 +; GCN-NEXT: s_mov_b32 s11, s10 +; GCN-NEXT: s_addc_u32 s3, s3, s10 ; GCN-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GCN-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GCN-NEXT: v_trunc_f32_e32 v1, v1 ; GCN-NEXT: v_madmk_f32 v0, v1, 0xcf800000, v0 ; GCN-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GCN-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GCN-NEXT: s_xor_b64 s[12:13], s[2:3], s[10:11] +; GCN-NEXT: s_mov_b32 s7, 0xf000 +; GCN-NEXT: v_mul_lo_u32 v2, s4, v1 +; GCN-NEXT: v_mul_hi_u32 v3, s4, v0 +; GCN-NEXT: v_mul_lo_u32 v5, s5, v0 +; GCN-NEXT: v_mul_lo_u32 v4, s4, v0 ; GCN-NEXT: s_mov_b32 s6, -1 -; GCN-NEXT: v_mul_lo_u32 v2, s2, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s2, v0 -; GCN-NEXT: v_mul_lo_u32 v5, s4, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s2, v0 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v5 ; GCN-NEXT: v_mul_hi_u32 v3, v0, v4 ; GCN-NEXT: v_mul_lo_u32 v5, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v6, v0, v2 -; GCN-NEXT: v_mul_hi_u32 v7, v1, v2 -; GCN-NEXT: v_mul_lo_u32 v2, v1, v2 -; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GCN-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc +; GCN-NEXT: v_mul_hi_u32 v7, v0, v2 ; GCN-NEXT: v_mul_lo_u32 v6, v1, v4 ; GCN-NEXT: v_mul_hi_u32 v4, v1, v4 +; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v5 +; GCN-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc +; GCN-NEXT: v_mul_hi_u32 v7, v1, v2 +; GCN-NEXT: v_mul_lo_u32 v2, v1, v2 ; GCN-NEXT: v_add_i32_e32 v3, vcc, v3, v6 ; GCN-NEXT: v_addc_u32_e32 v3, vcc, v5, v4, vcc ; GCN-NEXT: v_addc_u32_e32 v4, vcc, 0, v7, vcc @@ -967,12 +969,12 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GCN-NEXT: v_mul_lo_u32 v2, s2, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s2, v0 -; GCN-NEXT: v_mul_lo_u32 v4, s4, v0 -; GCN-NEXT: s_mov_b32 s4, s0 +; GCN-NEXT: v_mul_lo_u32 v2, s4, v1 +; GCN-NEXT: v_mul_hi_u32 v3, s4, v0 +; GCN-NEXT: v_mul_lo_u32 v4, s5, v0 +; GCN-NEXT: s_mov_b32 s5, s1 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v3 -; GCN-NEXT: v_mul_lo_u32 v3, s2, v0 +; GCN-NEXT: v_mul_lo_u32 v3, s4, v0 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; GCN-NEXT: v_mul_lo_u32 v6, v0, v2 ; GCN-NEXT: v_mul_hi_u32 v7, v0, v3 @@ -988,20 +990,18 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc ; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 ; GCN-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc -; GCN-NEXT: s_add_u32 s2, s10, s12 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GCN-NEXT: s_addc_u32 s3, s11, s12 ; GCN-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc -; GCN-NEXT: s_xor_b64 s[10:11], s[2:3], s[12:13] -; GCN-NEXT: v_mul_lo_u32 v2, s10, v1 -; GCN-NEXT: v_mul_hi_u32 v3, s10, v0 -; GCN-NEXT: v_mul_hi_u32 v4, s10, v1 -; GCN-NEXT: v_mul_hi_u32 v5, s11, v1 -; GCN-NEXT: v_mul_lo_u32 v1, s11, v1 +; GCN-NEXT: v_mul_lo_u32 v2, s12, v1 +; GCN-NEXT: v_mul_hi_u32 v3, s12, v0 +; GCN-NEXT: v_mul_hi_u32 v4, s12, v1 +; GCN-NEXT: v_mul_hi_u32 v5, s13, v1 +; GCN-NEXT: v_mul_lo_u32 v1, s13, v1 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v3, v2 ; GCN-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc -; GCN-NEXT: v_mul_lo_u32 v4, s11, v0 -; GCN-NEXT: v_mul_hi_u32 v0, s11, v0 +; GCN-NEXT: v_mul_lo_u32 v4, s13, v0 +; GCN-NEXT: v_mul_hi_u32 v0, s13, v0 +; GCN-NEXT: s_mov_b32 s4, s0 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; GCN-NEXT: v_addc_u32_e32 v0, vcc, v3, v0, vcc ; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc @@ -1013,9 +1013,9 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: v_mul_lo_u32 v0, s8, v0 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v1, v2 ; GCN-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, s11, v1 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s13, v1 ; GCN-NEXT: v_mov_b32_e32 v3, s9 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s10, v0 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s12, v0 ; GCN-NEXT: v_subb_u32_e64 v2, s[0:1], v2, v3, vcc ; GCN-NEXT: v_subrev_i32_e64 v4, s[0:1], s8, v0 ; GCN-NEXT: v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1] @@ -1030,7 +1030,7 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1] ; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 ; GCN-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v4, s11 +; GCN-NEXT: v_mov_b32_e32 v4, s13 ; GCN-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc ; GCN-NEXT: v_cmp_le_u32_e32 vcc, s9, v1 ; GCN-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc @@ -1042,10 +1042,10 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 % ; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GCN-NEXT: v_xor_b32_e32 v0, s12, v0 -; GCN-NEXT: v_xor_b32_e32 v1, s12, v1 -; GCN-NEXT: v_mov_b32_e32 v2, s12 -; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s12, v0 +; GCN-NEXT: v_xor_b32_e32 v0, s10, v0 +; GCN-NEXT: v_xor_b32_e32 v1, s10, v1 +; GCN-NEXT: v_mov_b32_e32 v2, s10 +; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s10, v0 ; GCN-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc ; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GCN-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll index 42bd2ff..9f539bd 100644 --- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll +++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll @@ -813,7 +813,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i64(ptr addrspace(1) %out, ; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 ; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 ; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3] glc dlc @@ -824,11 +825,12 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i64(ptr addrspace(1) %out, ; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v1, v0 ; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xffff, v0 -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, 0, 16, v2 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 +; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l +; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v2, 16, v3 +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 ; GFX11-TRUE16-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir index 8315708..6966c3d 100644 --- a/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir +++ b/llvm/test/CodeGen/AMDGPU/swdev502267-use-after-free-last-chance-recoloring-alloc-succeeds.mir @@ -34,26 +34,26 @@ body: | ; CHECK-NEXT: liveins: $vgpr0_vgpr1_vgpr2_vgpr3:0x00000000000000FF, $vgpr4_vgpr5_vgpr6_vgpr7:0x00000000000000FF ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: SI_SPILL_AV128_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3, %stack.1, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.1, align 4, addrspace 5) - ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) ; CHECK-NEXT: renamable $vgpr0 = V_FMA_F32_e64 0, $vgpr6, 0, $vgpr6, 0, killed $vgpr2, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: SI_SPILL_V128_SAVE $vgpr4_vgpr5_vgpr6_vgpr7, %stack.4, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.4, align 4, addrspace 5) + ; CHECK-NEXT: SI_SPILL_AV128_SAVE $vgpr4_vgpr5_vgpr6_vgpr7, %stack.4, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.4, align 4, addrspace 5) ; CHECK-NEXT: renamable $vgpr0 = V_TRUNC_F32_e32 killed $vgpr0, implicit $mode, implicit $exec - ; CHECK-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5) + ; CHECK-NEXT: SI_SPILL_AV32_SAVE killed $vgpr0, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5) ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF - ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) ; CHECK-NEXT: renamable $vgpr5 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, killed $vgpr7, 0, killed $vgpr5, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_V32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5) + ; CHECK-NEXT: renamable $vgpr0 = SI_SPILL_AV32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5) ; CHECK-NEXT: renamable $vgpr9 = COPY killed renamable $vgpr5 - ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_V128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5) - ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_AV128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) ; CHECK-NEXT: renamable $vgpr8 = nofpexcept V_FMA_F32_e64 1, killed $vgpr0, 0, killed $vgpr6, 0, killed $vgpr4, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: renamable $vgpr2_vgpr3 = COPY killed renamable $vgpr8_vgpr9 ; CHECK-NEXT: renamable $vgpr0 = IMPLICIT_DEF - ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_V128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5) - ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_AV128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_DIV_FIXUP_F32_e64 0, killed $vgpr0, 0, killed $vgpr4, 0, killed $vgpr6, 0, 0, implicit $mode, implicit $exec - ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_V128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5) - ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_V128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_AV128_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.4, align 4, addrspace 5) + ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = SI_SPILL_AV128_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s128) from %stack.2, align 4, addrspace 5) ; CHECK-NEXT: dead renamable $vgpr1 = V_FMA_F32_e64 0, killed $vgpr5, 0, $vgpr5, 0, killed $vgpr7, 0, 0, implicit $mode, implicit $exec ; CHECK-NEXT: dead renamable $vgpr4_vgpr5_vgpr6_vgpr7 = SCRATCH_LOAD_DWORDX4_SADDR %stack.0, 0, 0, implicit $exec, implicit $flat_scr :: (volatile load (s128), addrspace 5) ; CHECK-NEXT: renamable $vgpr4_vgpr5 = IMPLICIT_DEF diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll index f0829b5..c12265b 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll @@ -3924,37 +3924,37 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) { ; GFX7-SDAG: ; %bb.0: ; %entry ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[10:11], v[26:27] -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[12:13], v[28:29] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[18:19] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[22:23] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[24:25] -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e64 s[6:7], v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[10:11], v[26:27] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[16:17] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[12:13], v[28:29] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[20:21] -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[0:1], v[8:9] +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e64 s[6:7], v[0:1], v[8:9] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[12:13] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[22:23] +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[4:5], v[12:13] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[18:19] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -4028,37 +4028,37 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) { ; GFX8-SDAG: ; %bb.0: ; %entry ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[10:11], v[26:27] -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[12:13], v[28:29] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[18:19] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[22:23] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[24:25] -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e64 s[6:7], v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[10:11], v[26:27] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[16:17] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[12:13], v[28:29] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[20:21] -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[0:1], v[8:9] +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e64 s[6:7], v[0:1], v[8:9] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[12:13] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[22:23] +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[4:5], v[12:13] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[18:19] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -4132,47 +4132,49 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) { ; GFX9-SDAG: ; %bb.0: ; %entry ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32 -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[8:9], v[24:25] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[6:7], v[0:1], v[16:17] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[8:9], v[12:13], v[28:29] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[10:11], v[4:5], v[20:21] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[10:11], v[26:27] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[0:1], v[2:3], v[18:19] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[12:13] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[0:1], v[0:1], v[8:9] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[2:3], v[6:7], v[22:23] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[2:3], v[2:3], v[10:11] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[24:25] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[0:1], v[0:1], v[16:17] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[2:3], v[12:13], v[28:29] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[4:5], v[4:5], v[20:21] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[6:7], v[6:7], v[22:23] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[8:9], v[10:11], v[26:27] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[10:11], v[2:3], v[18:19] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v17, v21, v5, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v23, v7, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v27, v11, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[10:11] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v22, v6, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v26, v10, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[10:11] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[6:7] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[0:1], v[16:17], v[12:13] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e64 s[2:3], v[0:1], v[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v13, v17, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v12, v16, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[14:15], v[30:31] ; GFX9-SDAG-NEXT: s_nop 1 -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v31, v15, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v30, v14, vcc -; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v7, v31, v15, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v6, v30, v14, vcc +; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc ; GFX9-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[4:5] ; GFX9-SDAG-NEXT: s_nop 1 ; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc @@ -4242,49 +4244,49 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) { ; GFX10-SDAG: ; %bb.0: ; %entry ; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[8:9], v[24:25] +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[0:1], v[16:17] +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s6, v[6:7], v[22:23] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s4 ; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[10:11], v[26:27] ; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[2:3], v[18:19] -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[6:7], v[22:23] -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s6, v[8:9], v[24:25] -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s7, v[0:1], v[16:17] -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s8, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s5 +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s6 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc_lo -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[4:5], v[12:13] -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s6, v[0:1], v[8:9] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s6 -; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[14:15], v[30:31] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v15, v31, v15, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v14, v30, v14, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s6 +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[0:1], v[8:9] ; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[2:3], v[10:11] -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s5 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[0:1], v[4:5] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s4 -; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s5 +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[14:15], v[30:31] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc_lo +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[4:5], v[12:13] +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s4 +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[2:3], v[6:7] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s4 ; GFX10-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo @@ -4346,50 +4348,49 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) { ; GFX11-SDAG: ; %bb.0: ; %entry ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[8:9], v[24:25] +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[0:1], v[16:17] +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[6:7], v[22:23] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[10:11], v[26:27] ; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[18:19] -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[6:7], v[22:23] -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[8:9], v[24:25] -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s3, v[0:1], v[16:17] -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[4:5], v[20:21] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[4:5], v[20:21] ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[0:1], v[8:9] ; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[10:11] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[0:1], v[8:9] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[14:15], v[30:31] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[4:5], v[12:13] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[6:7], v[14:15] +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[6:7], v[14:15] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[0:1], v[4:5] -; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 -; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[6:7] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX11-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 ; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] @@ -4453,58 +4454,58 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) { ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[8:9], v[24:25] +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[0:1], v[16:17] +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[6:7], v[22:23] +; GFX12-SDAG-NEXT: s_wait_alu 0xfffd +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[10:11], v[26:27] ; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[18:19] -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[6:7], v[22:23] -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[8:9], v[24:25] -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s3, v[0:1], v[16:17] -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s4, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[4:5], v[20:21] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s5, v[4:5], v[20:21] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[10:11] -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s2, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 ; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[14:15], v[30:31] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[4:5], v[12:13] ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s1, v[6:7], v[14:15] +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[6:7], v[14:15] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX12-SDAG-NEXT: v_cmp_gt_i64_e64 s0, v[2:3], v[6:7] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX12-SDAG-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll index e674205..5056747 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll @@ -3924,37 +3924,37 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) { ; GFX7-SDAG: ; %bb.0: ; %entry ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[10:11], v[26:27] -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[12:13], v[28:29] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[18:19] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[22:23] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[24:25] -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e64 s[6:7], v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[10:11], v[26:27] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[16:17] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[12:13], v[28:29] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[20:21] -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[8:9] +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e64 s[6:7], v[0:1], v[8:9] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[12:13] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[22:23] +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[12:13] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[18:19] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -4028,37 +4028,37 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) { ; GFX8-SDAG: ; %bb.0: ; %entry ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[10:11], v[26:27] -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[12:13], v[28:29] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[18:19] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[22:23] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[24:25] -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e64 s[6:7], v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[10:11], v[26:27] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[16:17] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[12:13], v[28:29] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[20:21] -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[0:1], v[8:9] +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e64 s[6:7], v[0:1], v[8:9] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[12:13] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[22:23] +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[12:13] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[18:19] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -4132,47 +4132,49 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) { ; GFX9-SDAG: ; %bb.0: ; %entry ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32 -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[8:9], v[24:25] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[6:7], v[0:1], v[16:17] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[8:9], v[12:13], v[28:29] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[10:11], v[4:5], v[20:21] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[10:11], v[26:27] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[0:1], v[2:3], v[18:19] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[12:13] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[0:1], v[0:1], v[8:9] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[2:3], v[6:7], v[22:23] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[2:3], v[2:3], v[10:11] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[8:9], v[24:25] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[0:1], v[0:1], v[16:17] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[2:3], v[12:13], v[28:29] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], v[4:5], v[20:21] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[6:7], v[6:7], v[22:23] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[8:9], v[10:11], v[26:27] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[10:11], v[2:3], v[18:19] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v17, v21, v5, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v23, v7, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v27, v11, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[10:11] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v22, v6, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v26, v10, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[10:11] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[6:7] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[0:1], v[16:17], v[12:13] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e64 s[2:3], v[0:1], v[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v13, v17, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v12, v16, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[14:15], v[30:31] ; GFX9-SDAG-NEXT: s_nop 1 -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v31, v15, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v30, v14, vcc -; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[6:7], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v7, v31, v15, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v6, v30, v14, vcc +; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[4:5], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc ; GFX9-SDAG-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[4:5] ; GFX9-SDAG-NEXT: s_nop 1 ; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc @@ -4242,49 +4244,49 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) { ; GFX10-SDAG: ; %bb.0: ; %entry ; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[24:25] +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[0:1], v[16:17] +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s6, v[6:7], v[22:23] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s4 ; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[10:11], v[26:27] ; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[2:3], v[18:19] -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[6:7], v[22:23] -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s6, v[8:9], v[24:25] -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s7, v[0:1], v[16:17] -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s8, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s5 +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s6 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc_lo -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[12:13] -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s6, v[0:1], v[8:9] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s6 -; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[14:15], v[30:31] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v15, v31, v15, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v14, v30, v14, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s6 +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[0:1], v[8:9] ; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[2:3], v[10:11] -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s5 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[0:1], v[4:5] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s4 -; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s5 +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[14:15], v[30:31] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc_lo +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[12:13] +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s4 +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[2:3], v[6:7] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s4 ; GFX10-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3] ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo @@ -4346,50 +4348,49 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) { ; GFX11-SDAG: ; %bb.0: ; %entry ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[24:25] +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[0:1], v[16:17] +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s2, v[6:7], v[22:23] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[10:11], v[26:27] ; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[18:19] -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[6:7], v[22:23] -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s2, v[8:9], v[24:25] -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s3, v[0:1], v[16:17] -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[4:5], v[20:21] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[4:5], v[20:21] ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[0:1], v[8:9] ; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[10:11] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s2, v[0:1], v[8:9] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[14:15], v[30:31] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[12:13] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[6:7], v[14:15] +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[6:7], v[14:15] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[0:1], v[4:5] -; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 -; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[6:7] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX11-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 ; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] @@ -4453,58 +4454,58 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) { ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[24:25] +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[0:1], v[16:17] +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s2, v[6:7], v[22:23] +; GFX12-SDAG-NEXT: s_wait_alu 0xfffd +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[10:11], v[26:27] ; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[18:19] -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[6:7], v[22:23] -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s2, v[8:9], v[24:25] -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s3, v[0:1], v[16:17] -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s4, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[4:5], v[20:21] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s5, v[4:5], v[20:21] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[10:11] -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s2, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 ; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[14:15], v[30:31] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[12:13] ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s1, v[6:7], v[14:15] +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[6:7], v[14:15] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5] +; GFX12-SDAG-NEXT: v_cmp_lt_i64_e64 s0, v[2:3], v[6:7] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX12-SDAG-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll index 92993d0..ddae1b2 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll @@ -1718,7 +1718,7 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) { ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v3i16: ; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry ; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0 ; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-SDAG-TRUE16-NEXT: v_pk_max_u16 v0, v0, v1 ; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v0.h @@ -1751,7 +1751,7 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) { ; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0 ; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-SDAG-TRUE16-NEXT: v_pk_max_u16 v0, v0, v1 ; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v0.h @@ -3805,37 +3805,37 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) { ; GFX7-SDAG: ; %bb.0: ; %entry ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[10:11], v[26:27] -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[12:13], v[28:29] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[18:19] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[22:23] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[24:25] -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e64 s[6:7], v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[10:11], v[26:27] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[16:17] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[12:13], v[28:29] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[20:21] -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[8:9] +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e64 s[6:7], v[0:1], v[8:9] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[12:13] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[22:23] +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[12:13] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[18:19] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX7-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -3909,37 +3909,37 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) { ; GFX8-SDAG: ; %bb.0: ; %entry ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[10:11], v[26:27] -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[12:13], v[28:29] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[18:19] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[22:23] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[24:25] -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e64 s[6:7], v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[10:11], v[26:27] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[16:17] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[12:13], v[28:29] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[20:21] -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[8:9] +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e64 s[6:7], v[0:1], v[8:9] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[12:13] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[22:23] +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[12:13] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[18:19] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX8-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -4013,47 +4013,49 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) { ; GFX9-SDAG: ; %bb.0: ; %entry ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32 -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[24:25] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[6:7], v[0:1], v[16:17] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[8:9], v[12:13], v[28:29] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[10:11], v[4:5], v[20:21] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[10:11], v[26:27] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[2:3], v[18:19] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[12:13] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[8:9] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[2:3], v[6:7], v[22:23] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[2:3], v[2:3], v[10:11] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[24:25] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[16:17] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[2:3], v[12:13], v[28:29] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[4:5], v[4:5], v[20:21] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[6:7], v[6:7], v[22:23] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[8:9], v[10:11], v[26:27] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[10:11], v[2:3], v[18:19] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v17, v21, v5, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v23, v7, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v27, v11, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[10:11] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v22, v6, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v26, v10, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[10:11] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[6:7] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[16:17], v[12:13] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e64 s[2:3], v[0:1], v[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v13, v17, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v12, v16, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[14:15], v[30:31] ; GFX9-SDAG-NEXT: s_nop 1 -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v31, v15, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v30, v14, vcc -; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v7, v31, v15, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v6, v30, v14, vcc +; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc ; GFX9-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[4:5] ; GFX9-SDAG-NEXT: s_nop 1 ; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc @@ -4123,49 +4125,49 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) { ; GFX10-SDAG: ; %bb.0: ; %entry ; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[8:9], v[24:25] +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[0:1], v[16:17] +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s6, v[6:7], v[22:23] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s4 ; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[10:11], v[26:27] ; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[2:3], v[18:19] -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[6:7], v[22:23] -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s6, v[8:9], v[24:25] -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s7, v[0:1], v[16:17] -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s8, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s5 +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s6 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc_lo -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[4:5], v[12:13] -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s6, v[0:1], v[8:9] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s6 -; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[14:15], v[30:31] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v15, v31, v15, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v14, v30, v14, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s6 +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[0:1], v[8:9] ; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[2:3], v[10:11] -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s5 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[0:1], v[4:5] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s4 -; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s5 +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[14:15], v[30:31] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc_lo +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[4:5], v[12:13] +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s4 +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[2:3], v[6:7] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s4 ; GFX10-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo @@ -4227,50 +4229,49 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) { ; GFX11-SDAG: ; %bb.0: ; %entry ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[8:9], v[24:25] +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[16:17] +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[6:7], v[22:23] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[10:11], v[26:27] ; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[18:19] -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[6:7], v[22:23] -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[8:9], v[24:25] -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s3, v[0:1], v[16:17] -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[4:5], v[20:21] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[4:5], v[20:21] ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[0:1], v[8:9] ; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[10:11] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[0:1], v[8:9] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[14:15], v[30:31] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[4:5], v[12:13] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[6:7], v[14:15] +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[6:7], v[14:15] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5] -; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 -; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[6:7] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX11-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 ; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] @@ -4334,58 +4335,58 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) { ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[8:9], v[24:25] +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[16:17] +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[6:7], v[22:23] +; GFX12-SDAG-NEXT: s_wait_alu 0xfffd +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[10:11], v[26:27] ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[18:19] -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[6:7], v[22:23] -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[8:9], v[24:25] -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s3, v[0:1], v[16:17] -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s4, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[4:5], v[20:21] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s5, v[4:5], v[20:21] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[10:11] -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s2, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[14:15], v[30:31] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[4:5], v[12:13] ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s1, v[6:7], v[14:15] +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[6:7], v[14:15] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX12-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[2:3], v[6:7] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX12-SDAG-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll index 2bcee37..e3a7ae5 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll @@ -3544,37 +3544,37 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) { ; GFX7-SDAG: ; %bb.0: ; %entry ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX7-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[10:11], v[26:27] -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[12:13], v[28:29] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[18:19] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[22:23] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[24:25] -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[10:11], v[26:27] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[16:17] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[12:13], v[28:29] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[20:21] -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[0:1], v[8:9] +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], v[0:1], v[8:9] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[22:23] +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[4:5], v[12:13] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[18:19] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX7-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5] -; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -3648,37 +3648,37 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) { ; GFX8-SDAG: ; %bb.0: ; %entry ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[10:11], v[26:27] -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[12:13], v[28:29] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[18:19] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[22:23] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[24:25] -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[10:11], v[26:27] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[16:17] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v11, v27, v11, s[4:5] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[12:13], v[28:29] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v10, v26, v10, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[20:21] -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[0:1], v[8:9] +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], v[0:1], v[8:9] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc -; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5] +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[22:23] +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[4:5], v[12:13] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[18:19] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v5, v13, v5, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[4:5] +; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[6:7] +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc ; GFX8-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5] -; GFX8-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[6:7] ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc ; GFX8-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) @@ -3752,47 +3752,49 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) { ; GFX9-SDAG: ; %bb.0: ; %entry ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32 -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[8:9], v[24:25] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], v[0:1], v[16:17] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[8:9], v[12:13], v[28:29] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[10:11], v[4:5], v[20:21] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[10:11], v[26:27] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[0:1], v[2:3], v[18:19] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[6:7] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[8:9] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s[10:11] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[0:1], v[0:1], v[8:9] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[2:3], v[6:7], v[22:23] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[0:1] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[0:1] -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[2:3], v[2:3], v[10:11] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[24:25] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[0:1], v[0:1], v[16:17] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[2:3], v[12:13], v[28:29] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], v[4:5], v[20:21] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], v[6:7], v[22:23] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[8:9], v[10:11], v[26:27] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[10:11], v[2:3], v[18:19] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v17, v21, v5, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v5, v23, v7, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v27, v11, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s[10:11] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v16, v20, v4, s[4:5] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v4, v22, v6, s[6:7] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v26, v10, s[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s[10:11] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[0:1], v[16:17], v[12:13] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e64 s[2:3], v[0:1], v[8:9] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v7, v13, v17, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[2:3] +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v6, v12, v16, s[0:1] +; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[2:3] +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[14:15], v[30:31] ; GFX9-SDAG-NEXT: s_nop 1 -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v31, v15, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v30, v14, vcc -; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[4:5] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s[2:3] -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc -; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v7, v31, v15, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v6, v30, v14, vcc +; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[4:5], v[6:7] +; GFX9-SDAG-NEXT: s_nop 1 +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc +; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc ; GFX9-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5] ; GFX9-SDAG-NEXT: s_nop 1 ; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc @@ -3862,49 +3864,49 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) { ; GFX10-SDAG: ; %bb.0: ; %entry ; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[8:9], v[24:25] +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[0:1], v[16:17] +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s6, v[6:7], v[22:23] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s4 ; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[10:11], v[26:27] ; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[2:3], v[18:19] -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[6:7], v[22:23] -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s6, v[8:9], v[24:25] -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s7, v[0:1], v[16:17] -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s8, v[12:13], v[28:29] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s5 +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s6 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc_lo -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[20:21] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s7 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s8 -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s7 -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13] -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s6, v[0:1], v[8:9] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s6 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s6 -; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[14:15], v[30:31] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v15, v31, v15, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v14, v30, v14, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s6 +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[0:1], v[8:9] ; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[2:3], v[10:11] -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s5 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s4 ; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s4 -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[0:1], v[4:5] -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s5 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s4 -; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s4 -; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s5 +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[14:15], v[30:31] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc_lo +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13] +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[6:7], v[14:15] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s4 +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX10-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[2:3], v[6:7] +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s4 +; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s4 ; GFX10-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc_lo ; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo @@ -3966,50 +3968,49 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) { ; GFX11-SDAG: ; %bb.0: ; %entry ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[8:9], v[24:25] +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[16:17] +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s2, v[6:7], v[22:23] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[10:11], v[26:27] ; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[18:19] -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[6:7], v[22:23] -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s2, v[8:9], v[24:25] -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s3, v[0:1], v[16:17] -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[12:13], v[28:29] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[4:5], v[20:21] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[4:5], v[20:21] ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[0:1], v[8:9] ; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[10:11] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s2, v[0:1], v[8:9] +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) ; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[14:15], v[30:31] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13] ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[6:7], v[14:15] +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[6:7], v[14:15] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[4:5] -; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 -; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX11-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[6:7] +; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX11-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] ; GFX11-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 ; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] @@ -4073,58 +4074,58 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) { ; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32 +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[8:9], v[24:25] +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[16:17] +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s2, v[6:7], v[22:23] +; GFX12-SDAG-NEXT: s_wait_alu 0xfffd +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v9, v25, v9 :: v_dual_cndmask_b32 v8, v24, v8 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s0 ; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[10:11], v[26:27] ; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[18:19] -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[6:7], v[22:23] -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s2, v[8:9], v[24:25] -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s3, v[0:1], v[16:17] -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s4, v[12:13], v[28:29] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s1 +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[4:5], v[20:21] +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s2 ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v11, v27, v11 :: v_dual_cndmask_b32 v10, v26, v10 -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s5, v[4:5], v[20:21] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v23, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v9, v25, v9, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v17, v1, s3 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v13, v29, v13, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s5 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v18, v2, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v12, v28, v12, s4 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s5 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, v24, v8, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v16, v0, s3 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v5, v21, v5, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v4, v20, v4, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v22, v6, s2 +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[10:11] -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s2, v[0:1], v[8:9] ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s1 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v11, v3, s0 ; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v10, v2, s0 -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s2 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s2 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s1 ; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0 ; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[14:15], v[30:31] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14 ; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13] ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s1, v[6:7], v[14:15] +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[6:7], v[14:15] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4 ; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s1 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, v14, v6, s0 ; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[0:1], v[4:5] -; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[6:7] -; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff -; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v1, v5, v1, s0 -; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v0, v4, v0, s0 +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5] +; GFX12-SDAG-NEXT: v_cmp_lt_u64_e64 s0, v[2:3], v[6:7] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd -; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2 +; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v1, v5, v1 :: v_dual_cndmask_b32 v0, v4, v0 +; GFX12-SDAG-NEXT: s_wait_alu 0xf1ff +; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 +; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v2, v6, v2, s0 ; GFX12-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3] ; GFX12-SDAG-NEXT: s_wait_alu 0xfffd ; GFX12-SDAG-NEXT: v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1 |