; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=SDAG ; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GISEL ; Test async mark/wait with global_load_lds and global loads ; This version uses wave barriers to enforce program order so that unrelated vmem ; instructions do not get reordered before reaching this point. define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) { ; SDAG-LABEL: interleaved_with_wave_barrier: ; SDAG: ; %bb.0: ; %entry ; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 ; SDAG-NEXT: s_wait_kmcnt 0x0 ; SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4 ; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5 ; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x54, v[0:1] ; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x54, v2 ; SDAG-NEXT: global_load_b32 v10, v[8:9], off offset:44 ; SDAG-NEXT: global_load_b32 v11, v[0:1], off offset:4 ; SDAG-NEXT: ; wave barrier ; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv ; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9] ; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2 ; SDAG-NEXT: ; wave barrier ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8 ; SDAG-NEXT: ; wave barrier ; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv ; SDAG-NEXT: ; wave barrier ; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48 ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: ; wait_asyncmark(1) ; SDAG-NEXT: s_wait_asynccnt 0x1 ; SDAG-NEXT: ds_load_b32 v3, v2 offset:84 ; SDAG-NEXT: ; wait_asyncmark(0) ; SDAG-NEXT: s_wait_asynccnt 0x0 ; SDAG-NEXT: ds_load_b32 v2, v2 offset:88 ; SDAG-NEXT: s_wait_loadcnt 0x2 ; SDAG-NEXT: v_add_nc_u32_e32 v4, v11, v10 ; SDAG-NEXT: s_wait_loadcnt_dscnt 0x101 ; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; SDAG-NEXT: v_add3_u32 v0, v4, v3, v0 ; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0 ; SDAG-NEXT: v_add3_u32 v0, v0, v1, v2 ; SDAG-NEXT: global_store_b32 v[6:7], v0, off ; SDAG-NEXT: s_set_pc_i64 s[30:31] ; ; GISEL-LABEL: interleaved_with_wave_barrier: ; GISEL: ; %bb.0: ; %entry ; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 ; GISEL-NEXT: s_wait_kmcnt 0x0 ; GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4 ; GISEL-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6 ; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x54, v0 ; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x54, v2 ; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo ; GISEL-NEXT: global_load_b32 v10, v[8:9], off offset:44 ; GISEL-NEXT: global_load_b32 v11, v[0:1], off offset:4 ; GISEL-NEXT: ; wave barrier ; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv ; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8 ; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo ; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2 ; GISEL-NEXT: ; wave barrier ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:8 ; GISEL-NEXT: ; wave barrier ; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv ; GISEL-NEXT: ; wave barrier ; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48 ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: ; wait_asyncmark(1) ; GISEL-NEXT: s_wait_asynccnt 0x1 ; GISEL-NEXT: ds_load_b32 v3, v2 offset:84 ; GISEL-NEXT: ; wait_asyncmark(0) ; GISEL-NEXT: s_wait_asynccnt 0x0 ; GISEL-NEXT: ds_load_b32 v2, v2 offset:88 ; GISEL-NEXT: s_wait_loadcnt 0x2 ; GISEL-NEXT: v_add_nc_u32_e32 v6, v11, v10 ; GISEL-NEXT: s_wait_loadcnt_dscnt 0x101 ; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GISEL-NEXT: v_add3_u32 v0, v6, v3, v0 ; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 ; GISEL-NEXT: v_add3_u32 v0, v0, v1, v2 ; GISEL-NEXT: global_store_b32 v[4:5], v0, off ; GISEL-NEXT: s_set_pc_i64 s[30:31] entry: ; First batch: global load, global load, async global-to-LDS %bar_gep11 = getelementptr i32, ptr addrspace(1) %bar, i32 11 %bar_v11 = load i32, ptr addrspace(1) %bar_gep11 %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1 %foo_v1 = load i32, ptr addrspace(1) %foo_gep1 %lds_gep21 = getelementptr i32, ptr addrspace(3) %lds, i32 21 %bar_gep21 = getelementptr i32, ptr addrspace(1) %foo, i32 21 call void @llvm.amdgcn.wave.barrier() call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21) call void @llvm.amdgcn.wave.barrier() call void @llvm.amdgcn.asyncmark() ; Second batch: global load, async global-to-LDS, global load %foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2 %foo_v2 = load i32, ptr addrspace(1) %foo_gep2 %bar_gep22 = getelementptr i32, ptr addrspace(1) %bar, i32 22 %lds_gep22 = getelementptr i32, ptr addrspace(3) %lds, i32 22 call void @llvm.amdgcn.wave.barrier() call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep22, ptr addrspace(3) %lds_gep22, i32 4, i32 u0x23) call void @llvm.amdgcn.wave.barrier() %bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12 %bar_v12 = load i32, ptr addrspace(1) %bar_gep12 call void @llvm.amdgcn.asyncmark() ; Wait for first async mark and read from LDS call void @llvm.amdgcn.wait.asyncmark(i16 1) %lds_val21 = load i32, ptr addrspace(3) %lds_gep21 ; Wait for the next async mark. ; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one. call void @llvm.amdgcn.wait.asyncmark(i16 0) %lds_val22 = load i32, ptr addrspace(3) %lds_gep22 %sum1 = add i32 %foo_v1, %bar_v11 %sum2 = add i32 %sum1, %lds_val21 %sum3 = add i32 %sum2, %foo_v2 ; Finally a loadcnt(0) for %bar_v12, which was not included in the async mark that followed it. %sum4 = add i32 %sum3, %bar_v12 %sum5 = add i32 %sum4, %lds_val22 store i32 %sum5, ptr addrspace(1) %out ret void } ; A perfect loop that is unlikely to exist in real life. It uses only async ; operations, and result in waits that exactly match the stream of ; those outstanding operations. define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) { ; SDAG-LABEL: test_pipelined_loop: ; SDAG: ; %bb.0: ; %prolog ; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; SDAG-NEXT: s_clause 0x1 ; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv ; SDAG-NEXT: s_load_b32 s3, s[4:5], 0x44 nv ; SDAG-NEXT: s_wait_kmcnt 0x0 ; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; SDAG-NEXT: s_add_co_i32 s6, s2, 4 ; SDAG-NEXT: s_mov_b32 s7, s2 ; SDAG-NEXT: v_mov_b32_e32 v2, s6 ; SDAG-NEXT: s_mov_b32 s6, 2 ; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv ; SDAG-NEXT: v_mov_b32_e32 v1, 4 ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: global_load_async_to_lds_b32 v2, v1, s[0:1] offset:4 nv ; SDAG-NEXT: v_mov_b32_e32 v1, 0 ; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8 ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: .LBB1_1: ; %loop_body ; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 ; SDAG-NEXT: s_add_co_i32 s8, s7, 8 ; SDAG-NEXT: s_add_co_i32 s6, s6, 1 ; SDAG-NEXT: v_mov_b32_e32 v2, s8 ; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv ; SDAG-NEXT: v_mov_b32_e32 v2, s7 ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: ; wait_asyncmark(2) ; SDAG-NEXT: s_wait_asynccnt 0x2 ; SDAG-NEXT: s_add_co_i32 s7, s7, 4 ; SDAG-NEXT: s_cmp_lt_i32 s6, s3 ; SDAG-NEXT: ds_load_b32 v2, v2 ; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4 ; SDAG-NEXT: s_wait_dscnt 0x0 ; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v2 ; SDAG-NEXT: s_cbranch_scc1 .LBB1_1 ; SDAG-NEXT: ; %bb.2: ; %epilog ; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2 ; SDAG-NEXT: ; wait_asyncmark(1) ; SDAG-NEXT: s_wait_asynccnt 0x1 ; SDAG-NEXT: s_add_co_i32 s0, s0, -8 ; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0 ; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv ; SDAG-NEXT: ds_load_b32 v0, v0 ; SDAG-NEXT: ; wait_asyncmark(0) ; SDAG-NEXT: s_wait_dscnt 0x0 ; SDAG-NEXT: s_wait_asynccnt 0x0 ; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0 ; SDAG-NEXT: s_wait_kmcnt 0x0 ; SDAG-NEXT: global_store_b32 v2, v0, s[0:1] ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_pipelined_loop: ; GISEL: ; %bb.0: ; %prolog ; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; GISEL-NEXT: s_clause 0x1 ; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv ; GISEL-NEXT: s_load_b32 s3, s[4:5], 0x44 nv ; GISEL-NEXT: s_mov_b32 s7, 2 ; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v2, 4 ; GISEL-NEXT: s_wait_kmcnt 0x0 ; GISEL-NEXT: v_mov_b32_e32 v1, s2 ; GISEL-NEXT: s_add_co_u32 s6, s2, 4 ; GISEL-NEXT: s_mov_b32 s8, s2 ; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv ; GISEL-NEXT: v_mov_b32_e32 v1, s6 ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: s_mov_b32 s6, 0 ; GISEL-NEXT: global_load_async_to_lds_b32 v1, v2, s[0:1] offset:4 nv ; GISEL-NEXT: s_add_co_u32 s0, s0, 8 ; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0 ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: .LBB1_1: ; %loop_body ; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1 ; GISEL-NEXT: s_add_co_u32 s9, s8, 8 ; GISEL-NEXT: s_add_co_i32 s7, s7, 1 ; GISEL-NEXT: v_mov_b32_e32 v1, s9 ; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv ; GISEL-NEXT: v_mov_b32_e32 v1, s8 ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: ; wait_asyncmark(2) ; GISEL-NEXT: s_wait_asynccnt 0x2 ; GISEL-NEXT: ds_load_b32 v1, v1 ; GISEL-NEXT: s_wait_dscnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s9, v1 ; GISEL-NEXT: s_add_co_i32 s6, s6, s9 ; GISEL-NEXT: s_add_co_u32 s0, s0, 4 ; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0 ; GISEL-NEXT: s_add_co_u32 s8, s8, 4 ; GISEL-NEXT: s_cmp_lt_i32 s7, s3 ; GISEL-NEXT: s_cbranch_scc1 .LBB1_1 ; GISEL-NEXT: ; %bb.2: ; %epilog ; GISEL-NEXT: s_lshl_b32 s0, s3, 2 ; GISEL-NEXT: ; wait_asyncmark(1) ; GISEL-NEXT: s_wait_asynccnt 0x1 ; GISEL-NEXT: s_add_co_u32 s0, s2, s0 ; GISEL-NEXT: v_mov_b32_e32 v1, 0 ; GISEL-NEXT: s_add_co_u32 s0, s0, -8 ; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GISEL-NEXT: v_mov_b32_e32 v0, s0 ; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv ; GISEL-NEXT: ds_load_b32 v0, v0 ; GISEL-NEXT: ; wait_asyncmark(0) ; GISEL-NEXT: s_wait_dscnt 0x0 ; GISEL-NEXT: s_wait_asynccnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s2, v0 ; GISEL-NEXT: s_add_co_i32 s2, s6, s2 ; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GISEL-NEXT: v_mov_b32_e32 v0, s2 ; GISEL-NEXT: s_wait_kmcnt 0x0 ; GISEL-NEXT: global_store_b32 v1, v0, s[0:1] ; GISEL-NEXT: s_endpgm prolog: ; Load first iteration call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20) call void @llvm.amdgcn.asyncmark() ; Load second iteration %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1 %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1 call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20) call void @llvm.amdgcn.asyncmark() br label %loop_body loop_body: %i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ] %sum = phi i32 [ 0, %prolog ], [ %sum_i, %loop_body ] ; Load next iteration %lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i %foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20) call void @llvm.amdgcn.asyncmark() ; Wait for iteration i-2 and process call void @llvm.amdgcn.wait.asyncmark(i16 2) %lds_idx = sub i32 %i, 2 %lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx %lds_val = load i32, ptr addrspace(3) %lds_gep_read %sum_i = add i32 %sum, %lds_val %i.next = add i32 %i, 1 %cmp = icmp slt i32 %i.next, %n br i1 %cmp, label %loop_body, label %epilog epilog: ; Process remaining iterations call void @llvm.amdgcn.wait.asyncmark(i16 1) %lds_n_2 = sub i32 %n, 2 %lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2 %lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2 %sum_e2 = add i32 %sum_i, %lds_val_n_2 %out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2 call void @llvm.amdgcn.wait.asyncmark(i16 0) %lds_n_1 = sub i32 %n, 1 %lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1 %lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1 %sum_e1 = add i32 %sum_e2, %lds_val_n_1 store i32 %sum_e2, ptr addrspace(1) %bar ret void } ; Software pipelined loop with async global-to-LDS and global loads define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) { ; SDAG-LABEL: test_pipelined_loop_with_global: ; SDAG: ; %bb.0: ; %prolog ; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; SDAG-NEXT: s_clause 0x1 ; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv ; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv ; SDAG-NEXT: v_mov_b32_e32 v0, 0 ; SDAG-NEXT: s_wait_kmcnt 0x0 ; SDAG-NEXT: s_load_b32 s6, s[8:9], 0x0 ; SDAG-NEXT: s_load_b32 s7, s[0:1], 0x0 ; SDAG-NEXT: v_mov_b32_e32 v1, s10 ; SDAG-NEXT: s_add_co_i32 s11, s10, 4 ; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; SDAG-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s11 ; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv ; SDAG-NEXT: s_clause 0x2 ; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4 ; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4 ; SDAG-NEXT: s_wait_xcnt 0x0 ; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8 ; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8 ; SDAG-NEXT: s_wait_kmcnt 0x0 ; SDAG-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s7 ; SDAG-NEXT: s_mov_b64 s[6:7], s[2:3] ; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv ; SDAG-NEXT: s_wait_loadcnt 0x0 ; SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2 ; SDAG-NEXT: s_mov_b32 s8, 2 ; SDAG-NEXT: s_mov_b32 s9, s10 ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: .LBB2_1: ; %loop_body ; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1 ; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; SDAG-NEXT: s_add_co_i32 s12, s9, 8 ; SDAG-NEXT: s_wait_loadcnt 0x0 ; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12 ; SDAG-NEXT: v_mov_b32_e32 v8, v3 ; SDAG-NEXT: s_clause 0x1 ; SDAG-NEXT: global_load_b32 v3, v0, s[4:5] ; SDAG-NEXT: global_load_b32 v4, v0, s[0:1] ; SDAG-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2 ; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv ; SDAG-NEXT: v_mov_b32_e32 v9, s9 ; SDAG-NEXT: ; asyncmark ; SDAG-NEXT: ; wait_asyncmark(2) ; SDAG-NEXT: s_wait_asynccnt 0x2 ; SDAG-NEXT: s_add_co_i32 s8, s8, 1 ; SDAG-NEXT: s_add_co_i32 s9, s9, 4 ; SDAG-NEXT: ds_load_b32 v9, v9 ; SDAG-NEXT: v_mov_b32_e32 v5, v1 ; SDAG-NEXT: s_cmp_lt_i32 s8, s11 ; SDAG-NEXT: s_wait_xcnt 0x0 ; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4 ; SDAG-NEXT: s_add_nc_u64 s[4:5], s[4:5], 4 ; SDAG-NEXT: s_wait_dscnt 0x0 ; SDAG-NEXT: v_add_nc_u32_e32 v9, v10, v9 ; SDAG-NEXT: global_store_b32 v0, v9, s[6:7] ; SDAG-NEXT: s_wait_xcnt 0x0 ; SDAG-NEXT: s_add_nc_u64 s[6:7], s[6:7], 4 ; SDAG-NEXT: s_cbranch_scc1 .LBB2_1 ; SDAG-NEXT: ; %bb.2: ; %epilog ; SDAG-NEXT: s_add_co_i32 s0, s11, -2 ; SDAG-NEXT: ; wait_asyncmark(1) ; SDAG-NEXT: s_wait_asynccnt 0x1 ; SDAG-NEXT: s_lshl2_add_u32 s1, s0, s10 ; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; SDAG-NEXT: v_dual_add_nc_u32 v2, v8, v7 :: v_dual_mov_b32 v0, s1 ; SDAG-NEXT: ds_load_b32 v1, v0 ; SDAG-NEXT: s_wait_dscnt 0x0 ; SDAG-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_add_nc_u32 v1, v2, v1 ; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset ; SDAG-NEXT: ; wait_asyncmark(0) ; SDAG-NEXT: s_wait_asynccnt 0x0 ; SDAG-NEXT: ds_load_b32 v0, v0 offset:4 ; SDAG-NEXT: s_wait_loadcnt 0x0 ; SDAG-NEXT: s_wait_xcnt 0x0 ; SDAG-NEXT: v_add_nc_u32_e32 v1, v3, v4 ; SDAG-NEXT: s_wait_dscnt 0x0 ; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) ; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0 ; SDAG-NEXT: global_store_b32 v5, v0, s[2:3] offset:4 scale_offset ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_pipelined_loop_with_global: ; GISEL: ; %bb.0: ; %prolog ; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 ; GISEL-NEXT: s_clause 0x2 ; GISEL-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv ; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv ; GISEL-NEXT: s_load_b32 s11, s[4:5], 0x44 nv ; GISEL-NEXT: v_mov_b32_e32 v0, 0 ; GISEL-NEXT: s_wait_kmcnt 0x0 ; GISEL-NEXT: s_load_b32 s14, s[8:9], 0x0 ; GISEL-NEXT: s_load_b32 s15, s[0:1], 0x0 ; GISEL-NEXT: v_mov_b32_e32 v1, s10 ; GISEL-NEXT: s_add_co_u32 s6, s10, 4 ; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v3, s6 ; GISEL-NEXT: s_mov_b64 s[6:7], s[2:3] ; GISEL-NEXT: s_clause 0x2 ; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4 ; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4 ; GISEL-NEXT: s_wait_xcnt 0x0 ; GISEL-NEXT: s_add_co_u32 s0, s0, 8 ; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0 ; GISEL-NEXT: s_add_co_u32 s4, s8, 8 ; GISEL-NEXT: s_add_co_ci_u32 s5, s9, 0 ; GISEL-NEXT: global_load_async_to_lds_b32 v3, v4, s[8:9] offset:4 nv ; GISEL-NEXT: s_wait_loadcnt 0x1 ; GISEL-NEXT: v_readfirstlane_b32 s12, v1 ; GISEL-NEXT: s_wait_loadcnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s13, v2 ; GISEL-NEXT: s_mov_b32 s8, 2 ; GISEL-NEXT: s_mov_b32 s9, s10 ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: s_mov_b32 s17, s12 ; GISEL-NEXT: s_mov_b32 s19, s13 ; GISEL-NEXT: .LBB2_1: ; %loop_body ; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1 ; GISEL-NEXT: s_add_co_u32 s16, s9, 8 ; GISEL-NEXT: s_clause 0x1 ; GISEL-NEXT: global_load_b32 v1, v0, s[4:5] ; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] ; GISEL-NEXT: v_mov_b32_e32 v3, s16 ; GISEL-NEXT: s_mov_b32 s18, s17 ; GISEL-NEXT: s_wait_kmcnt 0x0 ; GISEL-NEXT: s_add_co_i32 s15, s14, s15 ; GISEL-NEXT: s_add_co_i32 s8, s8, 1 ; GISEL-NEXT: s_mov_b32 s16, s19 ; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[4:5] offset:4 nv ; GISEL-NEXT: v_mov_b32_e32 v3, s9 ; GISEL-NEXT: ; asyncmark ; GISEL-NEXT: ; wait_asyncmark(2) ; GISEL-NEXT: s_wait_asynccnt 0x2 ; GISEL-NEXT: s_mov_b32 s14, s12 ; GISEL-NEXT: ds_load_b32 v3, v3 ; GISEL-NEXT: s_wait_dscnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s17, v3 ; GISEL-NEXT: s_add_co_i32 s15, s15, s17 ; GISEL-NEXT: s_wait_xcnt 0x0 ; GISEL-NEXT: s_add_co_u32 s0, s0, 4 ; GISEL-NEXT: v_mov_b32_e32 v3, s15 ; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0 ; GISEL-NEXT: s_add_co_u32 s4, s4, 4 ; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0 ; GISEL-NEXT: s_mov_b32 s15, s13 ; GISEL-NEXT: global_store_b32 v0, v3, s[6:7] ; GISEL-NEXT: s_wait_xcnt 0x0 ; GISEL-NEXT: s_add_co_u32 s6, s6, 4 ; GISEL-NEXT: s_add_co_ci_u32 s7, s7, 0 ; GISEL-NEXT: s_add_co_u32 s9, s9, 4 ; GISEL-NEXT: s_cmp_lt_i32 s8, s11 ; GISEL-NEXT: s_wait_loadcnt 0x1 ; GISEL-NEXT: v_readfirstlane_b32 s17, v1 ; GISEL-NEXT: s_wait_loadcnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s19, v2 ; GISEL-NEXT: s_cbranch_scc1 .LBB2_1 ; GISEL-NEXT: ; %bb.2: ; %epilog ; GISEL-NEXT: s_add_co_i32 s0, s11, -2 ; GISEL-NEXT: ; wait_asyncmark(1) ; GISEL-NEXT: s_wait_asynccnt 0x1 ; GISEL-NEXT: s_lshl_b32 s1, s0, 2 ; GISEL-NEXT: s_add_co_i32 s4, s18, s16 ; GISEL-NEXT: s_add_co_u32 s1, s10, s1 ; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GISEL-NEXT: v_mov_b32_e32 v0, s1 ; GISEL-NEXT: ds_load_b32 v1, v0 ; GISEL-NEXT: s_wait_dscnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s1, v1 ; GISEL-NEXT: v_mov_b32_e32 v1, s0 ; GISEL-NEXT: s_add_co_i32 s1, s4, s1 ; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GISEL-NEXT: v_mov_b32_e32 v2, s1 ; GISEL-NEXT: s_add_co_i32 s1, s17, s19 ; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset ; GISEL-NEXT: ; wait_asyncmark(0) ; GISEL-NEXT: s_wait_asynccnt 0x0 ; GISEL-NEXT: ds_load_b32 v0, v0 offset:4 ; GISEL-NEXT: s_wait_dscnt 0x0 ; GISEL-NEXT: v_readfirstlane_b32 s0, v0 ; GISEL-NEXT: s_add_co_i32 s0, s1, s0 ; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GISEL-NEXT: v_mov_b32_e32 v0, s0 ; GISEL-NEXT: global_store_b32 v1, v0, s[2:3] offset:4 scale_offset ; GISEL-NEXT: s_endpgm prolog: ; Load first iteration %v0 = load i32, ptr addrspace(1) %foo %g0 = load i32, ptr addrspace(1) %bar call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20) call void @llvm.amdgcn.asyncmark() ; Load second iteration %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1 %v1 = load i32, ptr addrspace(1) %foo_gep1 %bar_gep1 = getelementptr i32, ptr addrspace(1) %bar, i32 1 %g1 = load i32, ptr addrspace(1) %bar_gep1 %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1 call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20) call void @llvm.amdgcn.asyncmark() br label %loop_body loop_body: %i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ] %prev_v = phi i32 [ %v0, %prolog ], [ %v1, %loop_body ] %prev_g = phi i32 [ %g0, %prolog ], [ %g1, %loop_body ] %v1_phi = phi i32 [ %v1, %prolog ], [ %cur_v, %loop_body ] %g1_phi = phi i32 [ %g1, %prolog ], [ %cur_g, %loop_body ] ; Load next iteration %foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i %cur_v = load i32, ptr addrspace(1) %foo_gep_cur %bar_gep_cur = getelementptr i32, ptr addrspace(1) %bar, i32 %i %cur_g = load i32, ptr addrspace(1) %bar_gep_cur %lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20) call void @llvm.amdgcn.asyncmark() ; Wait for iteration i-2 and process call void @llvm.amdgcn.wait.asyncmark(i16 2) %lds_idx = sub i32 %i, 2 %lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx %lds_val = load i32, ptr addrspace(3) %lds_gep_read %sum1 = add i32 %prev_v, %prev_g %sum2 = add i32 %sum1, %lds_val %out_gep = getelementptr i32, ptr addrspace(1) %out, i32 %lds_idx store i32 %sum2, ptr addrspace(1) %out_gep %i.next = add i32 %i, 1 %cmp = icmp slt i32 %i.next, %n br i1 %cmp, label %loop_body, label %epilog epilog: ; Process remaining iterations call void @llvm.amdgcn.wait.asyncmark(i16 1) %lds_n_2 = sub i32 %n, 2 %lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2 %lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2 %sum_e1 = add i32 %v1_phi, %g1_phi %sum_e2 = add i32 %sum_e1, %lds_val_n_2 %out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2 store i32 %sum_e2, ptr addrspace(1) %out_gep_e1 call void @llvm.amdgcn.wait.asyncmark(i16 0) %lds_n_1 = sub i32 %n, 1 %lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1 %lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1 %sum_e3 = add i32 %cur_v, %cur_g %sum_e4 = add i32 %sum_e3, %lds_val_n_1 %out_gep_e2 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_1 store i32 %sum_e4, ptr addrspace(1) %out_gep_e2 ret void }