From 3d18c8cd265c0c0bf1d85226c4770a2dd0f86e8f Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 12 Feb 2024 18:29:55 -0800 Subject: [PATCH 001/284] [test] Replace aarch64-*-{eabi,gnueabi}{,hf} with aarch64 Similar to d39b4ce3ce8a3c256e01bdec2b140777a332a633 Using "eabi" or "gnueabi" for aarch64 targets is a common mistake and warned by Clang Driver. We want to avoid them elsewhere as well. Just use the common "aarch64" without other triple components. --- llvm/test/Analysis/CostModel/AArch64/cast.ll | 10 +++++----- llvm/test/Analysis/CostModel/AArch64/fptoi_sat.ll | 4 ++-- .../Analysis/LoopAccessAnalysis/number-of-memchecks.ll | 2 +- .../LoopAccessAnalysis/reverse-memcheck-bounds.ll | 2 +- llvm/test/CodeGen/AArch64/Redundantstore.ll | 2 +- .../CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll | 4 ++-- llvm/test/CodeGen/AArch64/aarch64-addv.ll | 4 ++-- llvm/test/CodeGen/AArch64/aarch64-tbz.ll | 4 ++-- llvm/test/CodeGen/AArch64/aarch64-unroll-and-jam.ll | 2 +- .../test/CodeGen/AArch64/aarch64-vcvtfp2fxs-combine.ll | 2 +- llvm/test/CodeGen/AArch64/arm64-build-vector.ll | 2 +- llvm/test/CodeGen/AArch64/arm64-movi.ll | 2 +- llvm/test/CodeGen/AArch64/arm64-popcnt.ll | 4 ++-- llvm/test/CodeGen/AArch64/arm64-rev.ll | 4 ++-- llvm/test/CodeGen/AArch64/asm-large-immediate.ll | 2 +- llvm/test/CodeGen/AArch64/bf16-shuffle.ll | 6 +++--- llvm/test/CodeGen/AArch64/bf16.ll | 2 +- llvm/test/CodeGen/AArch64/bitreverse.ll | 2 +- llvm/test/CodeGen/AArch64/cmpwithshort.ll | 2 +- .../AArch64/complex-deinterleaving-i16-add-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-i16-mul-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-i32-add-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-i32-mul-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-i64-add-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-i64-mul-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-i8-add-scalable.ll | 2 +- .../AArch64/complex-deinterleaving-splat-scalable.ll | 2 +- .../CodeGen/AArch64/complex-deinterleaving-splat.ll | 2 +- llvm/test/CodeGen/AArch64/cond-br-tuning.ll | 2 +- llvm/test/CodeGen/AArch64/consthoist-gep.ll | 2 +- llvm/test/CodeGen/AArch64/extbinopload.ll | 2 +- llvm/test/CodeGen/AArch64/extract-sext-zext.ll | 4 ++-- llvm/test/CodeGen/AArch64/fabs.ll | 8 ++++---- llvm/test/CodeGen/AArch64/faddsub.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fcmp.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fcopysign.ll | 4 ++-- llvm/test/CodeGen/AArch64/fcvt.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fdiv.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fence-singlethread.ll | 2 +- llvm/test/CodeGen/AArch64/fexplog.ll | 4 ++-- llvm/test/CodeGen/AArch64/fminimummaximum.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fminmax.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fmla.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fmul.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fneg.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll | 2 +- llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll | 2 +- llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll | 2 +- llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll | 2 +- llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_1op.ll | 2 +- llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_2op.ll | 2 +- llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_3op.ll | 2 +- llvm/test/CodeGen/AArch64/fpext.ll | 4 ++-- llvm/test/CodeGen/AArch64/fpow.ll | 4 ++-- llvm/test/CodeGen/AArch64/fpowi.ll | 4 ++-- llvm/test/CodeGen/AArch64/fptoi.ll | 8 ++++---- llvm/test/CodeGen/AArch64/fptrunc.ll | 4 ++-- llvm/test/CodeGen/AArch64/frem.ll | 8 ++++---- llvm/test/CodeGen/AArch64/frintn.ll | 2 +- llvm/test/CodeGen/AArch64/fsincos.ll | 4 ++-- llvm/test/CodeGen/AArch64/fsqrt.ll | 8 ++++---- llvm/test/CodeGen/AArch64/hints.ll | 2 +- llvm/test/CodeGen/AArch64/icmp.ll | 4 ++-- llvm/test/CodeGen/AArch64/insertextract.ll | 4 ++-- llvm/test/CodeGen/AArch64/intrinsics-memory-barrier.ll | 2 +- llvm/test/CodeGen/AArch64/itofp.ll | 8 ++++---- .../CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll | 2 +- llvm/test/CodeGen/AArch64/legalize-bug-bogus-cpu.ll | 2 +- llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll | 2 +- llvm/test/CodeGen/AArch64/merge-store-dependency.ll | 2 +- llvm/test/CodeGen/AArch64/merge-store.ll | 2 +- llvm/test/CodeGen/AArch64/min-max-combine.ll | 8 ++++---- llvm/test/CodeGen/AArch64/min-max.ll | 8 ++++---- llvm/test/CodeGen/AArch64/mul_pow2.ll | 4 ++-- llvm/test/CodeGen/AArch64/neon-extadd-extract.ll | 2 +- llvm/test/CodeGen/AArch64/neon_rbit.ll | 2 +- llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll | 4 ++-- llvm/test/CodeGen/AArch64/nzcv-save.ll | 2 +- llvm/test/CodeGen/AArch64/pacbti-module-attrs.ll | 2 +- llvm/test/CodeGen/AArch64/postra-mi-sched.ll | 2 +- llvm/test/CodeGen/AArch64/rbit.ll | 2 +- llvm/test/CodeGen/AArch64/rcpc3-sve.ll | 4 ++-- llvm/test/CodeGen/AArch64/rcpc3.ll | 4 ++-- llvm/test/CodeGen/AArch64/rem_crash.ll | 2 +- llvm/test/CodeGen/AArch64/rotate.ll | 2 +- llvm/test/CodeGen/AArch64/setcc_knownbits.ll | 2 +- .../CodeGen/AArch64/sls-stackprotector-outliner.ll | 2 +- llvm/test/CodeGen/AArch64/stack-probing-64k.ll | 4 ++-- llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll | 4 ++-- llvm/test/CodeGen/AArch64/stack-probing-sve.ll | 4 ++-- llvm/test/CodeGen/AArch64/stack-probing.ll | 4 ++-- llvm/test/CodeGen/AArch64/sve-fcopysign.ll | 4 ++-- llvm/test/CodeGen/AArch64/sve2-fcopysign.ll | 4 ++-- llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll | 2 +- llvm/test/CodeGen/AArch64/tbz-tbnz.ll | 2 +- llvm/test/CodeGen/AArch64/v3f-to-int.ll | 2 +- .../CodeGen/AArch64/v8.5a-neon-frint3264-intrinsic.ll | 2 +- .../AArch64/v8.5a-scalar-frint3264-intrinsic.ll | 2 +- llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll | 8 ++++---- llvm/test/CodeGen/AArch64/vecreduce-fadd.ll | 8 ++++---- llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll | 8 ++++---- llvm/test/CodeGen/AArch64/vecreduce-fmul.ll | 8 ++++---- llvm/test/CodeGen/AArch64/xar.ll | 4 ++-- llvm/test/Transforms/LICM/sink-foldable.ll | 2 +- .../LoopVectorize/AArch64/arbitrary-induction-step.ll | 2 +- .../LoopVectorize/AArch64/backedge-overflow.ll | 2 +- .../LoopVectorize/AArch64/interleaved_cost.ll | 2 +- 107 files changed, 200 insertions(+), 200 deletions(-) diff --git a/llvm/test/Analysis/CostModel/AArch64/cast.ll b/llvm/test/Analysis/CostModel/AArch64/cast.ll index 5dd37e83f479..0cd444f84985 100644 --- a/llvm/test/Analysis/CostModel/AArch64/cast.ll +++ b/llvm/test/Analysis/CostModel/AArch64/cast.ll @@ -1,9 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi -mattr=+sve -force-streaming-compatible-sve %s | FileCheck --check-prefixes=SVE,SVE128-NO-NEON %s -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi -mattr=+sve -aarch64-sve-vector-bits-min=256 %s | FileCheck --check-prefixes=SVE,FIXED-MIN-256 %s -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi -mattr=+sve -aarch64-sve-vector-bits-min=2048 %s | FileCheck --check-prefixes=SVE,FIXED-MIN-2048 %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -force-streaming-compatible-sve %s | FileCheck --check-prefixes=SVE,SVE128-NO-NEON %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=256 %s | FileCheck --check-prefixes=SVE,FIXED-MIN-256 %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=2048 %s | FileCheck --check-prefixes=SVE,FIXED-MIN-2048 %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/Analysis/CostModel/AArch64/fptoi_sat.ll b/llvm/test/Analysis/CostModel/AArch64/fptoi_sat.ll index a35242496848..e4e29143985b 100644 --- a/llvm/test/Analysis/CostModel/AArch64/fptoi_sat.ll +++ b/llvm/test/Analysis/CostModel/AArch64/fptoi_sat.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s -; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64-none-linux-gnueabi -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s +; RUN: opt -passes="print" 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll b/llvm/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll index c268cc55880c..d4287612399b 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/number-of-memchecks.ll @@ -1,7 +1,7 @@ ; RUN: opt -passes='print' -disable-output < %s 2>&1 | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -target triple = "aarch64--linux-gnueabi" +target triple = "aarch64" ; 3 reads and 3 writes should need 12 memchecks ; CHECK: function 'testf': diff --git a/llvm/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll b/llvm/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll index 86395eea96f5..1496e1b0be82 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/reverse-memcheck-bounds.ll @@ -12,7 +12,7 @@ ; } target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -target triple = "aarch64--linux-gnueabi" +target triple = "aarch64" ; CHECK: function 'f': ; CHECK: (Low: (20000 + %a) High: (60004 + %a)) diff --git a/llvm/test/CodeGen/AArch64/Redundantstore.ll b/llvm/test/CodeGen/AArch64/Redundantstore.ll index 6fec5573fdcb..229d6446c4e0 100644 --- a/llvm/test/CodeGen/AArch64/Redundantstore.ll +++ b/llvm/test/CodeGen/AArch64/Redundantstore.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -O3 -mtriple=aarch64-eabi | FileCheck %s +; RUN: llc < %s -O3 -mtriple=aarch64 | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @end_of_array = common global ptr null, align 8 diff --git a/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll b/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll index 122e187637c3..f2ed57ead074 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll @@ -6,8 +6,8 @@ ; The following tests use the balance-fp-ops feature, and should be independent of ; the target cpu. -; RUN: llc < %s -mtriple=aarch64-linux-gnueabi -mattr=+balance-fp-ops -aarch64-a57-fp-load-balancing-override=1 -aarch64-a57-fp-load-balancing-force-all -enable-misched=false -enable-post-misched=false | FileCheck %s --check-prefix CHECK --check-prefix CHECK-EVEN -; RUN: llc < %s -mtriple=aarch64-linux-gnueabi -mattr=+balance-fp-ops -aarch64-a57-fp-load-balancing-override=2 -aarch64-a57-fp-load-balancing-force-all -enable-misched=false -enable-post-misched=false | FileCheck %s --check-prefix CHECK --check-prefix CHECK-ODD +; RUN: llc < %s -mtriple=aarch64 -mattr=+balance-fp-ops -aarch64-a57-fp-load-balancing-override=1 -aarch64-a57-fp-load-balancing-force-all -enable-misched=false -enable-post-misched=false | FileCheck %s --check-prefix CHECK --check-prefix CHECK-EVEN +; RUN: llc < %s -mtriple=aarch64 -mattr=+balance-fp-ops -aarch64-a57-fp-load-balancing-override=2 -aarch64-a57-fp-load-balancing-force-all -enable-misched=false -enable-post-misched=false | FileCheck %s --check-prefix CHECK --check-prefix CHECK-ODD ; Test the AArch64A57FPLoadBalancing pass. This pass relies heavily on register allocation, so ; our test strategy is to: diff --git a/llvm/test/CodeGen/AArch64/aarch64-addv.ll b/llvm/test/CodeGen/AArch64/aarch64-addv.ll index 5b78b0d47731..b77d59134765 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-addv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-addv.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi -aarch64-neon-syntax=generic | FileCheck %s -check-prefixes=CHECK,SDAG -; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=aarch64-eabi -aarch64-neon-syntax=generic 2>&1 | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=generic | FileCheck %s -check-prefixes=CHECK,SDAG +; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=aarch64 -aarch64-neon-syntax=generic 2>&1 | FileCheck %s --check-prefixes=CHECK,GISEL ; Function Attrs: nounwind readnone declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>) diff --git a/llvm/test/CodeGen/AArch64/aarch64-tbz.ll b/llvm/test/CodeGen/AArch64/aarch64-tbz.ll index 28629a8c2f0d..4cf36337f9a2 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-tbz.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-tbz.ll @@ -1,5 +1,5 @@ -; RUN: llc -verify-machineinstrs -mtriple=aarch64-linux-gnueabi < %s | FileCheck %s -; RUN: llc -verify-machineinstrs -mtriple=aarch64-linux-gnueabi -cgp-verify-bfi-updates=true < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=aarch64 < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=aarch64 -cgp-verify-bfi-updates=true < %s | FileCheck %s ; CHECK-LABEL: test1 ; CHECK: tbz {{w[0-9]}}, #3, {{.LBB0_3}} diff --git a/llvm/test/CodeGen/AArch64/aarch64-unroll-and-jam.ll b/llvm/test/CodeGen/AArch64/aarch64-unroll-and-jam.ll index af5f6a9d6924..285c16a0b354 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-unroll-and-jam.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-unroll-and-jam.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='loop-unroll-and-jam' < %s -mcpu=cortex-a55 -mtriple=aarch64-none-linux-eabi -S | FileCheck %s +; RUN: opt -passes='loop-unroll-and-jam' < %s -mcpu=cortex-a55 -mtriple=aarch64 -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" diff --git a/llvm/test/CodeGen/AArch64/aarch64-vcvtfp2fxs-combine.ll b/llvm/test/CodeGen/AArch64/aarch64-vcvtfp2fxs-combine.ll index 463084e6fe6a..50f7d6d7ce9d 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-vcvtfp2fxs-combine.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-vcvtfp2fxs-combine.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-linux-eabi -o - | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s %struct.a= type { i64, i64, i64, i64 } diff --git a/llvm/test/CodeGen/AArch64/arm64-build-vector.ll b/llvm/test/CodeGen/AArch64/arm64-build-vector.ll index 68c56d765cbb..e4fbf0765dcd 100644 --- a/llvm/test/CodeGen/AArch64/arm64-build-vector.ll +++ b/llvm/test/CodeGen/AArch64/arm64-build-vector.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+fullfp16,+bf16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16,+bf16 | FileCheck %s ; Check that building a vector from floats doesn't insert an unnecessary ; copy for lane zero. diff --git a/llvm/test/CodeGen/AArch64/arm64-movi.ll b/llvm/test/CodeGen/AArch64/arm64-movi.ll index 8ec98b744297..c9074c2adbe3 100644 --- a/llvm/test/CodeGen/AArch64/arm64-movi.ll +++ b/llvm/test/CodeGen/AArch64/arm64-movi.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s ;==--------------------------------------------------------------------------== ; Tests for MOV-immediate implemented with ORR-immediate. diff --git a/llvm/test/CodeGen/AArch64/arm64-popcnt.ll b/llvm/test/CodeGen/AArch64/arm64-popcnt.ll index 599fac8c8034..f5ce73a36612 100644 --- a/llvm/test/CodeGen/AArch64/arm64-popcnt.ll +++ b/llvm/test/CodeGen/AArch64/arm64-popcnt.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s -; RUN: llc < %s -mtriple=aarch64-eabi -mattr -neon -aarch64-neon-syntax=apple | FileCheck -check-prefix=CHECK-NONEON %s -; RUN: llc < %s -mtriple=aarch64-eabi -mattr +cssc -aarch64-neon-syntax=apple | FileCheck -check-prefix=CHECK-CSSC %s +; RUN: llc < %s -mtriple=aarch64 -mattr -neon -aarch64-neon-syntax=apple | FileCheck -check-prefix=CHECK-NONEON %s +; RUN: llc < %s -mtriple=aarch64 -mattr +cssc -aarch64-neon-syntax=apple | FileCheck -check-prefix=CHECK-CSSC %s define i32 @cnt32_advsimd(i32 %x) nounwind readnone { ; CHECK-LABEL: cnt32_advsimd: diff --git a/llvm/test/CodeGen/AArch64/arm64-rev.ll b/llvm/test/CodeGen/AArch64/arm64-rev.ll index 5f61d9019ab4..f548a0e01fee 100644 --- a/llvm/test/CodeGen/AArch64/arm64-rev.ll +++ b/llvm/test/CodeGen/AArch64/arm64-rev.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc < %s -mtriple=aarch64-eabi -aarch64-neon-syntax=apple -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=apple -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i32 @test_rev_w(i32 %a) nounwind { ; CHECK-LABEL: test_rev_w: diff --git a/llvm/test/CodeGen/AArch64/asm-large-immediate.ll b/llvm/test/CodeGen/AArch64/asm-large-immediate.ll index 83690716a9e2..b45f22cc7073 100644 --- a/llvm/test/CodeGen/AArch64/asm-large-immediate.ll +++ b/llvm/test/CodeGen/AArch64/asm-large-immediate.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -no-integrated-as | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -no-integrated-as | FileCheck %s define void @test() { entry: diff --git a/llvm/test/CodeGen/AArch64/bf16-shuffle.ll b/llvm/test/CodeGen/AArch64/bf16-shuffle.ll index cf0b43807f4c..d59de3c56f4e 100644 --- a/llvm/test/CodeGen/AArch64/bf16-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/bf16-shuffle.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-eabi -mattr=+v8.6a,+neon < %s | FileCheck %s -; RUN: llc -mtriple=aarch64-eabi -mattr=+v8.6a,+neon,+bf16 < %s | FileCheck %s -; RUN: llc -mtriple=aarch64-eabi -mattr=+v8.6a,+neon,+fullfp16,+bf16 < %s | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon < %s | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon,+bf16 < %s | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon,+fullfp16,+bf16 < %s | FileCheck %s %struct.float16x4x2_t = type { [2 x <4 x bfloat>] } %struct.float16x8x2_t = type { [2 x <8 x bfloat>] } diff --git a/llvm/test/CodeGen/AArch64/bf16.ll b/llvm/test/CodeGen/AArch64/bf16.ll index 7a171c6702e1..d3911ae4c033 100644 --- a/llvm/test/CodeGen/AArch64/bf16.ll +++ b/llvm/test/CodeGen/AArch64/bf16.ll @@ -1,5 +1,5 @@ ; RUN: llc < %s -asm-verbose=0 -mtriple=arm64-eabi -mattr=+bf16 | FileCheck %s -; RUN: llc < %s -asm-verbose=0 -mtriple=aarch64-eabi -mattr=+bf16 | FileCheck %s +; RUN: llc < %s -asm-verbose=0 -mtriple=aarch64 -mattr=+bf16 | FileCheck %s ; test argument passing and simple load/store diff --git a/llvm/test/CodeGen/AArch64/bitreverse.ll b/llvm/test/CodeGen/AArch64/bitreverse.ll index be9f5b82c6c0..a6d3683a92a8 100644 --- a/llvm/test/CodeGen/AArch64/bitreverse.ll +++ b/llvm/test/CodeGen/AArch64/bitreverse.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s ; These tests just check that the plumbing is in place for @llvm.bitreverse. diff --git a/llvm/test/CodeGen/AArch64/cmpwithshort.ll b/llvm/test/CodeGen/AArch64/cmpwithshort.ll index 8dbfdae5df90..3dbf64a2f6f1 100644 --- a/llvm/test/CodeGen/AArch64/cmpwithshort.ll +++ b/llvm/test/CodeGen/AArch64/cmpwithshort.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -O3 -mtriple=aarch64-eabi | FileCheck %s +; RUN: llc < %s -O3 -mtriple=aarch64 | FileCheck %s define i16 @test_1cmp_signed_1(ptr %ptr1) { ; CHECK-LABEL: @test_1cmp_signed_1 diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-add-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-add-scalable.ll index 6f4f8d3ca2d3..001046f8f397 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-add-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-add-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to not transform as the type's minimum size is less than 128 bits. define @complex_add_v4i16( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll index b0a3e46c96c4..1b8a21b66ade 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to not transform as the type's minimum size is less than 128 bits. define @complex_mul_v4i16( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-add-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-add-scalable.ll index 3118d8669dc9..1ce480bbf3d8 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-add-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-add-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to transform define @complex_add_v4i32( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-mul-scalable.ll index 256ed10cad07..d88eef9800d7 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i32-mul-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to transform define @complex_mul_v4i32( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-add-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-add-scalable.ll index d9ec5fcd3bdc..0b59be9414fa 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-add-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-add-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to transform define @complex_add_v2i64( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-mul-scalable.ll index 2dec03b6f979..16e1f3e63dce 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i64-mul-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to transform define @complex_mul_v2i64( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i8-add-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i8-add-scalable.ll index e7ebd07fd731..b631486137e6 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i8-add-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i8-add-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve2 -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; Expected to not transform as the type's minimum size is less than 128 bits. define @complex_add_v8i8( %a, %b) { diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll index 0cbe2f46088e..2627f2a4fb5e 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+sve -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; a[i] * b[i] * (11.0 + 3.0.i); ; diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll index 8de2ac5a140c..ad9240b0922b 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s --mattr=+complxnum -o - | FileCheck %s -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" ; a[i] * b[i] * (11.0 + 3.0.i); diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll index dc00c41892ba..1e1af3e4a498 100644 --- a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll +++ b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -debugify-and-strip-all-safe < %s -O3 -mtriple=aarch64-eabi -verify-machineinstrs | FileCheck %s +; RUN: llc -debugify-and-strip-all-safe < %s -O3 -mtriple=aarch64 -verify-machineinstrs | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64-linaro-linux-gnueabi" diff --git a/llvm/test/CodeGen/AArch64/consthoist-gep.ll b/llvm/test/CodeGen/AArch64/consthoist-gep.ll index d109dade67f5..031ee353c0e8 100644 --- a/llvm/test/CodeGen/AArch64/consthoist-gep.ll +++ b/llvm/test/CodeGen/AArch64/consthoist-gep.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-unknown-linuxeabi -consthoist-gep %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 -consthoist-gep %s -o - | FileCheck %s %struct.blam = type { %struct.bar, %struct.bar.0, %struct.wobble, %struct.wombat, i8, i16, %struct.snork.2, %struct.foo, %struct.snork.3, %struct.wobble.4, %struct.quux, [9 x i16], %struct.spam, %struct.zot } %struct.bar = type { i8, i8, %struct.snork } diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll index 849fc7aa00a8..1f68c77611e1 100644 --- a/llvm/test/CodeGen/AArch64/extbinopload.ll +++ b/llvm/test/CodeGen/AArch64/extbinopload.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s define <4 x i16> @normal_load_v4i8(ptr %p) { ; CHECK-LABEL: normal_load_v4i8: diff --git a/llvm/test/CodeGen/AArch64/extract-sext-zext.ll b/llvm/test/CodeGen/AArch64/extract-sext-zext.ll index f566ebb4f208..ecb76d9320a2 100644 --- a/llvm/test/CodeGen/AArch64/extract-sext-zext.ll +++ b/llvm/test/CodeGen/AArch64/extract-sext-zext.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL -; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL +; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL +; RUN: llc -mtriple=aarch64 -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL define i64 @extract_v2i64(<2 x i64> %x, i32 %y) { ; CHECK-ISEL-LABEL: extract_v2i64: diff --git a/llvm/test/CodeGen/AArch64/fabs.ll b/llvm/test/CodeGen/AArch64/fabs.ll index c56c6a07357d..5462bc65fd34 100644 --- a/llvm/test/CodeGen/AArch64/fabs.ll +++ b/llvm/test/CodeGen/AArch64/fabs.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @fabs_f64(double %a) { ; CHECK-LABEL: fabs_f64: diff --git a/llvm/test/CodeGen/AArch64/faddsub.ll b/llvm/test/CodeGen/AArch64/faddsub.ll index c02e02d9257e..31389f5a77d6 100644 --- a/llvm/test/CodeGen/AArch64/faddsub.ll +++ b/llvm/test/CodeGen/AArch64/faddsub.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @fadd_f64(double %a, double %b) { ; CHECK-LABEL: fadd_f64: diff --git a/llvm/test/CodeGen/AArch64/fcmp.ll b/llvm/test/CodeGen/AArch64/fcmp.ll index 29138ba59cfe..0f02784aaf32 100644 --- a/llvm/test/CodeGen/AArch64/fcmp.ll +++ b/llvm/test/CodeGen/AArch64/fcmp.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @f64_double(double %a, double %b, double %d, double %e) { ; CHECK-LABEL: f64_double: diff --git a/llvm/test/CodeGen/AArch64/fcopysign.ll b/llvm/test/CodeGen/AArch64/fcopysign.ll index 4abd115da21c..a1c48bd943e2 100644 --- a/llvm/test/CodeGen/AArch64/fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/fcopysign.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @copysign_f64(double %a, double %b) { ; CHECK-SD-LABEL: copysign_f64: diff --git a/llvm/test/CodeGen/AArch64/fcvt.ll b/llvm/test/CodeGen/AArch64/fcvt.ll index ce38bebf7635..584174d6d064 100644 --- a/llvm/test/CodeGen/AArch64/fcvt.ll +++ b/llvm/test/CodeGen/AArch64/fcvt.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @ceil_f64(double %a) { ; CHECK-LABEL: ceil_f64: diff --git a/llvm/test/CodeGen/AArch64/fdiv.ll b/llvm/test/CodeGen/AArch64/fdiv.ll index b7a645bfb546..fa87c4fa2d16 100644 --- a/llvm/test/CodeGen/AArch64/fdiv.ll +++ b/llvm/test/CodeGen/AArch64/fdiv.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @fdiv_f64(double %a, double %b) { ; CHECK-LABEL: fdiv_f64: diff --git a/llvm/test/CodeGen/AArch64/fence-singlethread.ll b/llvm/test/CodeGen/AArch64/fence-singlethread.ll index f36d28926acb..eb77daa25241 100644 --- a/llvm/test/CodeGen/AArch64/fence-singlethread.ll +++ b/llvm/test/CodeGen/AArch64/fence-singlethread.ll @@ -1,7 +1,7 @@ ; RUN: llc -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s --check-prefix=LINUX ; RUN: llc -mtriple=aarch64-apple-ios %s -o - | FileCheck %s --check-prefix=IOS ; RUN: llc -mtriple=aarch64-apple-ios %s -o - -global-isel | FileCheck %s --check-prefix=IOS -; RUN: llc -mtriple=aarch64-linux-gnueabihf %s -filetype=obj -o %t +; RUN: llc -mtriple=aarch64 %s -filetype=obj -o %t ; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=OBJ ; OBJ-NOT: dmb diff --git a/llvm/test/CodeGen/AArch64/fexplog.ll b/llvm/test/CodeGen/AArch64/fexplog.ll index 2848a6bde320..e3c0ced79f07 100644 --- a/llvm/test/CodeGen/AArch64/fexplog.ll +++ b/llvm/test/CodeGen/AArch64/fexplog.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @exp_f64(double %a) { ; CHECK-LABEL: exp_f64: diff --git a/llvm/test/CodeGen/AArch64/fminimummaximum.ll b/llvm/test/CodeGen/AArch64/fminimummaximum.ll index 217e4e40a779..f0e946c13998 100644 --- a/llvm/test/CodeGen/AArch64/fminimummaximum.ll +++ b/llvm/test/CodeGen/AArch64/fminimummaximum.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-NOFP16-SD -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-NOFP16-GI -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-NOFP16-SD +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-NOFP16-GI +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16-GI define double @min_f64(double %a, double %b) { ; CHECK-LABEL: min_f64: diff --git a/llvm/test/CodeGen/AArch64/fminmax.ll b/llvm/test/CodeGen/AArch64/fminmax.ll index 1b92c462af14..cdf9973b49f4 100644 --- a/llvm/test/CodeGen/AArch64/fminmax.ll +++ b/llvm/test/CodeGen/AArch64/fminmax.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-NOFP16-SD -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-NOFP16-GI -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-NOFP16-SD +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-NOFP16-GI +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16-GI define double @min_f64(double %a, double %b) { ; CHECK-LABEL: min_f64: diff --git a/llvm/test/CodeGen/AArch64/fmla.ll b/llvm/test/CodeGen/AArch64/fmla.ll index 3ae2158a1886..339ade5fc707 100644 --- a/llvm/test/CodeGen/AArch64/fmla.ll +++ b/llvm/test/CodeGen/AArch64/fmla.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @fma_f64(double %a, double %b, double %c) { ; CHECK-LABEL: fma_f64: diff --git a/llvm/test/CodeGen/AArch64/fmul.ll b/llvm/test/CodeGen/AArch64/fmul.ll index 244c73bba2e9..fe84fe1f317a 100644 --- a/llvm/test/CodeGen/AArch64/fmul.ll +++ b/llvm/test/CodeGen/AArch64/fmul.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @fmul_f64(double %a, double %b) { ; CHECK-LABEL: fmul_f64: diff --git a/llvm/test/CodeGen/AArch64/fneg.ll b/llvm/test/CodeGen/AArch64/fneg.ll index fd427687c7a7..7805512cbf45 100644 --- a/llvm/test/CodeGen/AArch64/fneg.ll +++ b/llvm/test/CodeGen/AArch64/fneg.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @fabs_f64(double %a) { ; CHECK-LABEL: fabs_f64: diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll index f68691ac15c6..e9fbaf6f046d 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+fullfp16 | FileCheck %s declare half @llvm.aarch64.neon.fmulx.f16(half, half) declare <4 x half> @llvm.aarch64.neon.fmulx.v4f16(<4 x half>, <4 x half>) diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll index 62284ca56126..40d2d636b94b 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_1op.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+fullfp16 | FileCheck %s declare i64 @llvm.aarch64.neon.fcvtpu.i64.f16(half) declare i32 @llvm.aarch64.neon.fcvtpu.i32.f16(half) diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll index 111ddfeab705..36795f86e006 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_2op.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+fullfp16 | FileCheck %s declare half @llvm.aarch64.sisd.fabd.f16(half, half) declare half @llvm.aarch64.neon.fmax.f16(half, half) diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll index ba8e616eeeb0..9a52e214d82d 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+neon,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+neon,+fullfp16 | FileCheck %s define dso_local half @t_vfmah_f16(half %a, half %b, half %c) { ; CHECK-LABEL: t_vfmah_f16: diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_1op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_1op.ll index becbbdd45434..58cbc2953dbc 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_1op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_1op.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+fullfp16 | FileCheck %s declare <4 x half> @llvm.nearbyint.v4f16(<4 x half>) declare <8 x half> @llvm.nearbyint.v8f16(<8 x half>) diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_2op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_2op.ll index 1674d8627920..e29919a8a9d4 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_2op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_2op.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+fullfp16 | FileCheck %s declare <4 x half> @llvm.aarch64.neon.fmulx.v4f16(<4 x half>, <4 x half>) declare <8 x half> @llvm.aarch64.neon.fmulx.v8f16(<8 x half>, <8 x half>) diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_3op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_3op.ll index c8a33a6cf5c2..8d52d2a428d0 100644 --- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_3op.ll +++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_vector_3op.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.2a,+fullfp16 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.2a,+fullfp16 | FileCheck %s declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>) declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>) diff --git a/llvm/test/CodeGen/AArch64/fpext.ll b/llvm/test/CodeGen/AArch64/fpext.ll index 9635b888f0da..db1105d613cd 100644 --- a/llvm/test/CodeGen/AArch64/fpext.ll +++ b/llvm/test/CodeGen/AArch64/fpext.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @fpext_f32_f64(float %a) { ; CHECK-LABEL: fpext_f32_f64: diff --git a/llvm/test/CodeGen/AArch64/fpow.ll b/llvm/test/CodeGen/AArch64/fpow.ll index a55c0dbffae2..1dd5450c271c 100644 --- a/llvm/test/CodeGen/AArch64/fpow.ll +++ b/llvm/test/CodeGen/AArch64/fpow.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @pow_f64(double %a, double %b) { ; CHECK-LABEL: pow_f64: diff --git a/llvm/test/CodeGen/AArch64/fpowi.ll b/llvm/test/CodeGen/AArch64/fpowi.ll index 677d2e0416ae..b496c7d15eef 100644 --- a/llvm/test/CodeGen/AArch64/fpowi.ll +++ b/llvm/test/CodeGen/AArch64/fpowi.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @powi_f64(double %a, i32 %b) { ; CHECK-LABEL: powi_f64: diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll index 23ba85d54c7a..251719c1e3b4 100644 --- a/llvm/test/CodeGen/AArch64/fptoi.ll +++ b/llvm/test/CodeGen/AArch64/fptoi.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 ; CHECK-GI-FP16: warning: Instruction selection used fallback path for fptos_v2f16_v2i16 ; CHECK-GI-FP16-NEXT: warning: Instruction selection used fallback path for fptou_v2f16_v2i16 diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll index 813fa0396e78..9425988af834 100644 --- a/llvm/test/CodeGen/AArch64/fptrunc.ll +++ b/llvm/test/CodeGen/AArch64/fptrunc.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define float @fptrunc_f64_f32(double %a) { ; CHECK-LABEL: fptrunc_f64_f32: diff --git a/llvm/test/CodeGen/AArch64/frem.ll b/llvm/test/CodeGen/AArch64/frem.ll index eb26128d6469..03caf0a33eb4 100644 --- a/llvm/test/CodeGen/AArch64/frem.ll +++ b/llvm/test/CodeGen/AArch64/frem.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @frem_f64(double %a, double %b) { ; CHECK-LABEL: frem_f64: diff --git a/llvm/test/CodeGen/AArch64/frintn.ll b/llvm/test/CodeGen/AArch64/frintn.ll index 2dc03db39a1a..782ba1d2dc78 100644 --- a/llvm/test/CodeGen/AArch64/frintn.ll +++ b/llvm/test/CodeGen/AArch64/frintn.ll @@ -1,4 +1,4 @@ -; RUN: llc -mtriple=aarch64-eabi -mattr=+fullfp16 %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 %s -o - | FileCheck %s ; The llvm.aarch64.neon.frintn intrinsic should be auto-upgraded to the ; target-independent roundeven intrinsic. diff --git a/llvm/test/CodeGen/AArch64/fsincos.ll b/llvm/test/CodeGen/AArch64/fsincos.ll index aef0b2e29243..2c76d969d6ef 100644 --- a/llvm/test/CodeGen/AArch64/fsincos.ll +++ b/llvm/test/CodeGen/AArch64/fsincos.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define double @sin_f64(double %a) { ; CHECK-LABEL: sin_f64: diff --git a/llvm/test/CodeGen/AArch64/fsqrt.ll b/llvm/test/CodeGen/AArch64/fsqrt.ll index 76930e76399f..683544a69ebe 100644 --- a/llvm/test/CodeGen/AArch64/fsqrt.ll +++ b/llvm/test/CodeGen/AArch64/fsqrt.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define double @sqrt_f64(double %a) { ; CHECK-LABEL: sqrt_f64: diff --git a/llvm/test/CodeGen/AArch64/hints.ll b/llvm/test/CodeGen/AArch64/hints.ll index f23c7b00f224..61a3fa4f07c7 100644 --- a/llvm/test/CodeGen/AArch64/hints.ll +++ b/llvm/test/CodeGen/AArch64/hints.ll @@ -1,4 +1,4 @@ -; RUN: llc -mtriple aarch64-eabi -o - %s | FileCheck %s +; RUN: llc -mtriple=aarch64 -o - %s | FileCheck %s declare void @llvm.aarch64.hint(i32) nounwind diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll index 789bc99810ae..2e8c93a00a0d 100644 --- a/llvm/test/CodeGen/AArch64/icmp.ll +++ b/llvm/test/CodeGen/AArch64/icmp.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i64 @i64_i64(i64 %a, i64 %b, i64 %d, i64 %e) { ; CHECK-LABEL: i64_i64: diff --git a/llvm/test/CodeGen/AArch64/insertextract.ll b/llvm/test/CodeGen/AArch64/insertextract.ll index 794abca1ae42..6074d44cb03d 100644 --- a/llvm/test/CodeGen/AArch64/insertextract.ll +++ b/llvm/test/CodeGen/AArch64/insertextract.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI ; CHECK-GI: warning: Instruction selection used fallback path for insert_v2f64_c ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for insert_v3f64_c diff --git a/llvm/test/CodeGen/AArch64/intrinsics-memory-barrier.ll b/llvm/test/CodeGen/AArch64/intrinsics-memory-barrier.ll index 61939977c4d7..ea4205f51fee 100644 --- a/llvm/test/CodeGen/AArch64/intrinsics-memory-barrier.ll +++ b/llvm/test/CodeGen/AArch64/intrinsics-memory-barrier.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -O=3 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -O=3 | FileCheck %s define void @test() { ; CHECK: dmb sy diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll index fa1ab61a6216..85689b6b2a60 100644 --- a/llvm/test/CodeGen/AArch64/itofp.ll +++ b/llvm/test/CodeGen/AArch64/itofp.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 ; CHECK-GI: warning: Instruction selection used fallback path for stofp_v3i8_v3f64 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for utofp_v3i8_v3f64 diff --git a/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll b/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll index cf638356dfdf..91cf605613b9 100644 --- a/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll +++ b/llvm/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -aarch64-neon-syntax=apple -aarch64-enable-stp-suppress=false -verify-machineinstrs -asm-verbose=false | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=apple -aarch64-enable-stp-suppress=false -verify-machineinstrs -asm-verbose=false | FileCheck %s ; CHECK-LABEL: test_strd_sturd: ; CHECK-NEXT: stp d0, d1, [x0, #-8] diff --git a/llvm/test/CodeGen/AArch64/legalize-bug-bogus-cpu.ll b/llvm/test/CodeGen/AArch64/legalize-bug-bogus-cpu.ll index a96a3c5f4881..a949abb3ee5a 100644 --- a/llvm/test/CodeGen/AArch64/legalize-bug-bogus-cpu.ll +++ b/llvm/test/CodeGen/AArch64/legalize-bug-bogus-cpu.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mcpu=bogus +; RUN: llc < %s -mtriple=aarch64 -mcpu=bogus ; Fix the bug in PR20557. Set mcpu to a bogus name, llc will crash in type ; legalization. diff --git a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll index 871bc3b0a323..23011dfc2d4f 100644 --- a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll +++ b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll @@ -1,4 +1,4 @@ -; RUN: llc %s -mtriple=aarch64-eabi -stop-after=finalize-isel -o - | FileCheck --check-prefix=MIR %s +; RUN: llc %s -mtriple=aarch64 -stop-after=finalize-isel -o - | FileCheck --check-prefix=MIR %s ; Ensure the scoped AA metadata is still retained after store merging. diff --git a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll index fc5813b99ea5..3c42987ade20 100644 --- a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll +++ b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mcpu cortex-a53 -mtriple=aarch64-eabi | FileCheck %s --check-prefix=A53 +; RUN: llc < %s -mcpu cortex-a53 -mtriple=aarch64 | FileCheck %s --check-prefix=A53 ; PR26827 - Merge stores causes wrong dependency. %struct1 = type { ptr, ptr, i32, i32, i16, i16, ptr, ptr } diff --git a/llvm/test/CodeGen/AArch64/merge-store.ll b/llvm/test/CodeGen/AArch64/merge-store.ll index f7201030da2c..b93d0c3bc960 100644 --- a/llvm/test/CodeGen/AArch64/merge-store.ll +++ b/llvm/test/CodeGen/AArch64/merge-store.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mcpu=cyclone -mattr=+slow-misaligned-128store | FileCheck %s --check-prefixes=CHECK,SPLITTING -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=-slow-misaligned-128store | FileCheck %s --check-prefixes=CHECK,MISALIGNED +; RUN: llc < %s -mtriple=aarch64 -mattr=-slow-misaligned-128store | FileCheck %s --check-prefixes=CHECK,MISALIGNED @g0 = external dso_local global <3 x float>, align 16 @g1 = external dso_local global <3 x float>, align 4 diff --git a/llvm/test/CodeGen/AArch64/min-max-combine.ll b/llvm/test/CodeGen/AArch64/min-max-combine.ll index 535d2bad2967..5111f838b73a 100644 --- a/llvm/test/CodeGen/AArch64/min-max-combine.ll +++ b/llvm/test/CodeGen/AArch64/min-max-combine.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s --check-prefixes=CHECK-ISEL -; RUN: llc -mtriple=aarch64-eabi %s -o - -mattr=cssc | FileCheck %s --check-prefixes=CHECK-CSSC -; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK-GLOBAL -; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - -mattr=cssc | FileCheck %s --check-prefixes=CHECK-CSSC +; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s --check-prefixes=CHECK-ISEL +; RUN: llc -mtriple=aarch64 %s -o - -mattr=cssc | FileCheck %s --check-prefixes=CHECK-CSSC +; RUN: llc -mtriple=aarch64 -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK-GLOBAL +; RUN: llc -mtriple=aarch64 -global-isel %s -o - -mattr=cssc | FileCheck %s --check-prefixes=CHECK-CSSC ; These tests check for @llvm.smax, @llvm.smin combines. diff --git a/llvm/test/CodeGen/AArch64/min-max.ll b/llvm/test/CodeGen/AArch64/min-max.ll index 8914406f1db7..0d02f1e7b548 100644 --- a/llvm/test/CodeGen/AArch64/min-max.ll +++ b/llvm/test/CodeGen/AArch64/min-max.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL -; RUN: llc -mtriple=aarch64-eabi %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL-CSSC -; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL -; RUN: llc -mtriple=aarch64-eabi -global-isel %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL-CSSC +; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL +; RUN: llc -mtriple=aarch64 %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-ISEL-CSSC +; RUN: llc -mtriple=aarch64 -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL +; RUN: llc -mtriple=aarch64 -global-isel %s -o - -mattr=+cssc | FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL-CSSC ; These tests just check that the plumbing is in place for @llvm.smax, @llvm.umax, ; @llvm.smin, @llvm.umin. diff --git a/llvm/test/CodeGen/AArch64/mul_pow2.ll b/llvm/test/CodeGen/AArch64/mul_pow2.ll index e16ee40c8dcb..90e560af4465 100644 --- a/llvm/test/CodeGen/AArch64/mul_pow2.ll +++ b/llvm/test/CodeGen/AArch64/mul_pow2.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-eabi | FileCheck %s -; RUN: llc < %s -mtriple=aarch64-eabi -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=GISEL +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=GISEL ; Convert mul x, pow2 to shift. ; Convert mul x, pow2 +/- 1 to shift + add/sub. diff --git a/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll b/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll index d79c07205555..b3692f8f46fb 100644 --- a/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll +++ b/llvm/test/CodeGen/AArch64/neon-extadd-extract.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc < %s -mtriple aarch64-none-eabi -o - | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s define <4 x i16> @addls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-LABEL: addls_v8i8_0: diff --git a/llvm/test/CodeGen/AArch64/neon_rbit.ll b/llvm/test/CodeGen/AArch64/neon_rbit.ll index 0daaf72f93dd..e66aca7613cb 100644 --- a/llvm/test/CodeGen/AArch64/neon_rbit.ll +++ b/llvm/test/CodeGen/AArch64/neon_rbit.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-eabi -mattr=+fullfp16 %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 %s -o - | FileCheck %s ; The llvm.aarch64_neon_rbit intrinsic should be auto-upgraded to the ; target-independent bitreverse intrinsic. diff --git a/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll b/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll index 4fe16d8195c0..b7dde881291b 100644 --- a/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll +++ b/llvm/test/CodeGen/AArch64/no-quad-ldp-stp.ll @@ -1,5 +1,5 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+slow-paired-128 -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,SLOW -; RUN: llc < %s -mtriple=aarch64-eabi -mcpu=exynos-m3 -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,FAST +; RUN: llc < %s -mtriple=aarch64 -mattr=+slow-paired-128 -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,SLOW +; RUN: llc < %s -mtriple=aarch64 -mcpu=exynos-m3 -verify-machineinstrs -asm-verbose=false | FileCheck %s --check-prefixes=CHECK,FAST ; CHECK-LABEL: test_nopair_st ; SLOW: str diff --git a/llvm/test/CodeGen/AArch64/nzcv-save.ll b/llvm/test/CodeGen/AArch64/nzcv-save.ll index 9bc4ccf2787a..c40e529ccab1 100644 --- a/llvm/test/CodeGen/AArch64/nzcv-save.ll +++ b/llvm/test/CodeGen/AArch64/nzcv-save.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-eabi | FileCheck %s +; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64 | FileCheck %s ; DAG ends up with two uses for the flags from an ADCS node, which means they ; must be saved for later. diff --git a/llvm/test/CodeGen/AArch64/pacbti-module-attrs.ll b/llvm/test/CodeGen/AArch64/pacbti-module-attrs.ll index 1858f3b566f8..ba4772178211 100644 --- a/llvm/test/CodeGen/AArch64/pacbti-module-attrs.ll +++ b/llvm/test/CodeGen/AArch64/pacbti-module-attrs.ll @@ -1,4 +1,4 @@ -;; RUN: llc -mtriple=aarch64-eabi -mattr=+v8.5a %s -o - | FileCheck %s +;; RUN: llc -mtriple=aarch64 -mattr=+v8.5a %s -o - | FileCheck %s declare i32 @g(i32) #5 diff --git a/llvm/test/CodeGen/AArch64/postra-mi-sched.ll b/llvm/test/CodeGen/AArch64/postra-mi-sched.ll index 7688973a1f2f..5abc06bd6ba6 100644 --- a/llvm/test/CodeGen/AArch64/postra-mi-sched.ll +++ b/llvm/test/CodeGen/AArch64/postra-mi-sched.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -O3 -mtriple=aarch64-eabi -mcpu=cortex-a53 | FileCheck %s +; RUN: llc < %s -O3 -mtriple=aarch64 -mcpu=cortex-a53 | FileCheck %s ; With cortex-a53, each of fmul and fcvt have latency of 6 cycles. After the ; pre-RA MI scheduler, fmul, fcvt and fdiv will be consecutive. The top-down diff --git a/llvm/test/CodeGen/AArch64/rbit.ll b/llvm/test/CodeGen/AArch64/rbit.ll index 288a25bd65e3..8c457b4480f8 100644 --- a/llvm/test/CodeGen/AArch64/rbit.ll +++ b/llvm/test/CodeGen/AArch64/rbit.ll @@ -1,4 +1,4 @@ -; RUN: llc -mtriple=aarch64-eabi %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s ; The llvm.aarch64.rbit intrinsic should be auto-upgraded to the ; target-independent bitreverse intrinsic. diff --git a/llvm/test/CodeGen/AArch64/rcpc3-sve.ll b/llvm/test/CodeGen/AArch64/rcpc3-sve.ll index b9d93942d0f1..47e338151749 100644 --- a/llvm/test/CodeGen/AArch64/rcpc3-sve.ll +++ b/llvm/test/CodeGen/AArch64/rcpc3-sve.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+v8.9a -mattr=+sve -mattr=+rcpc3 < %s | FileCheck %s -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+v8.9a -mattr=+sve < %s | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.9a -mattr=+sve -mattr=+rcpc3 < %s | FileCheck %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.9a -mattr=+sve < %s | FileCheck %s ; Show what happens with RCPC3 for extract/insert into SVE vectors. ; Currently there is no RCPC3 codegen expected for this. diff --git a/llvm/test/CodeGen/AArch64/rcpc3.ll b/llvm/test/CodeGen/AArch64/rcpc3.ll index d41613953b3c..45770337c132 100644 --- a/llvm/test/CodeGen/AArch64/rcpc3.ll +++ b/llvm/test/CodeGen/AArch64/rcpc3.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+v8.9a -mattr=+rcpc3 < %s | FileCheck --check-prefixes=BOTH,RCPC3 %s -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+v8.9a < %s | FileCheck --check-prefixes=BOTH,NO-RCPC3 %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.9a -mattr=+rcpc3 < %s | FileCheck --check-prefixes=BOTH,RCPC3 %s +; RUN: llc -mtriple=aarch64 -mattr=+v8.9a < %s | FileCheck --check-prefixes=BOTH,NO-RCPC3 %s define hidden <2 x i64> @test_ldap1_2xi64_lane0(ptr nocapture noundef readonly %a, <2 x i64> noundef %b) local_unnamed_addr { ; diff --git a/llvm/test/CodeGen/AArch64/rem_crash.ll b/llvm/test/CodeGen/AArch64/rem_crash.ll index f9cf6d58370c..38b46eae9f53 100644 --- a/llvm/test/CodeGen/AArch64/rem_crash.ll +++ b/llvm/test/CodeGen/AArch64/rem_crash.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi +; RUN: llc < %s -mtriple=aarch64 define i8 @test_minsize_uu8(i8 %x) minsize optsize { entry: diff --git a/llvm/test/CodeGen/AArch64/rotate.ll b/llvm/test/CodeGen/AArch64/rotate.ll index 8d52b6dd3a79..7b4808e7d466 100644 --- a/llvm/test/CodeGen/AArch64/rotate.ll +++ b/llvm/test/CodeGen/AArch64/rotate.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64--linux-gnueabihf | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s ;; This used to cause a backend crash about not being able to ;; select ROTL. Make sure if generates the basic ushr/shl. diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll index 9e9c814be026..46b714d8e5fb 100644 --- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll +++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3 -; RUN: llc < %s -mtriple=aarch64-none-eabi | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s define i1 @load_bv_v4i8(i1 zeroext %a) { ; CHECK-LABEL: load_bv_v4i8: diff --git a/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll b/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll index 9e5e55512df8..5f3b1503b46b 100644 --- a/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll +++ b/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll @@ -5,7 +5,7 @@ ; inserted at a point where LR is live. target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" -target triple = "aarch64-arm-none-eabi" +target triple = "aarch64" define hidden void @_ZTv0_n24_N2C6D1Ev(ptr %this) minsize sspreq "target-features"="+harden-sls-retbr" { ; CHECK-LABEL: _ZTv0_n24_N2C6D1Ev: diff --git a/llvm/test/CodeGen/AArch64/stack-probing-64k.ll b/llvm/test/CodeGen/AArch64/stack-probing-64k.ll index 0d64e7378b55..5f833e32fb8c 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-64k.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-64k.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs -enable-post-misched=false | FileCheck %s -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs -enable-post-misched=false -global-isel | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -enable-post-misched=false | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -enable-post-misched=false -global-isel | FileCheck %s ; Tests for prolog sequences for stack probing, when using a 64KiB stack guard. diff --git a/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll b/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll index a3b8df487ed4..d9ad1045506c 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs | FileCheck %s -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s ; Dynamically-sized allocation, needs a loop which can handle any size at ; runtime. The final iteration of the loop will temporarily put SP below the diff --git a/llvm/test/CodeGen/AArch64/stack-probing-sve.ll b/llvm/test/CodeGen/AArch64/stack-probing-sve.ll index 03a9220ebfdd..1ad78709d501 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-sve.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-sve.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs | FileCheck %s -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s ; Test prolog sequences for stack probing when SVE objects are involved. diff --git a/llvm/test/CodeGen/AArch64/stack-probing.ll b/llvm/test/CodeGen/AArch64/stack-probing.ll index df5408de5bab..8fc90cfd71b1 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs -enable-post-misched=false | FileCheck %s -; RUN: llc -mtriple aarch64-none-eabi < %s -verify-machineinstrs -enable-post-misched=false -global-isel | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -enable-post-misched=false | FileCheck %s +; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -enable-post-misched=false -global-isel | FileCheck %s ; Tests for prolog sequences for stack probing, when using a 4KiB stack guard. diff --git a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll index d2315844dc2f..f15807597ac2 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple aarch64-eabi -mattr=+sve -o - | FileCheck --check-prefixes=CHECK,CHECK-NO-EXTEND-ROUND %s -; RUN: llc < %s -mtriple aarch64-eabi -mattr=+sve --combiner-vector-fcopysign-extend-round -o - | FileCheck --check-prefixes=CHECK,CHECK-EXTEND-ROUND %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+sve -o - | FileCheck --check-prefixes=CHECK,CHECK-NO-EXTEND-ROUND %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+sve --combiner-vector-fcopysign-extend-round -o - | FileCheck --check-prefixes=CHECK,CHECK-EXTEND-ROUND %s target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" ;============ v2f32 diff --git a/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll index 7f65997fda7a..14cc8cd26826 100644 --- a/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple aarch64-eabi -mattr=+sve2 -o - | FileCheck --check-prefixes=CHECK,CHECK_NO_EXTEND_ROUND %s -; RUN: llc < %s -mtriple aarch64-eabi -mattr=+sve2 --combiner-vector-fcopysign-extend-round -o - | FileCheck --check-prefixes=CHECK,CHECK_EXTEND_ROUND %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -o - | FileCheck --check-prefixes=CHECK,CHECK_NO_EXTEND_ROUND %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 --combiner-vector-fcopysign-extend-round -o - | FileCheck --check-prefixes=CHECK,CHECK_EXTEND_ROUND %s target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll b/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll index 54c200ec4c3d..675380787af4 100644 --- a/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll +++ b/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll @@ -1,4 +1,4 @@ -; RUN: llc <%s -mtriple=aarch64-eabi -verify-machine-dom-info | FileCheck %s +; RUN: llc <%s -mtriple=aarch64 -verify-machine-dom-info | FileCheck %s ; CHECK-LABEL: test: ; CHECK-LABEL: %cond.false12.i diff --git a/llvm/test/CodeGen/AArch64/tbz-tbnz.ll b/llvm/test/CodeGen/AArch64/tbz-tbnz.ll index 1edea229cfe8..d301a380dbb8 100644 --- a/llvm/test/CodeGen/AArch64/tbz-tbnz.ll +++ b/llvm/test/CodeGen/AArch64/tbz-tbnz.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -O1 -mtriple=aarch64-eabi -aarch64-enable-cond-br-tune=false | FileCheck %s +; RUN: llc < %s -O1 -mtriple=aarch64 -aarch64-enable-cond-br-tune=false | FileCheck %s declare void @t() diff --git a/llvm/test/CodeGen/AArch64/v3f-to-int.ll b/llvm/test/CodeGen/AArch64/v3f-to-int.ll index a3c9c8fbe0a3..f6553b6acec9 100644 --- a/llvm/test/CodeGen/AArch64/v3f-to-int.ll +++ b/llvm/test/CodeGen/AArch64/v3f-to-int.ll @@ -1,4 +1,4 @@ -; RUN: llc -mtriple=aarch64--linux-eabi %s -o - | FileCheck %s +; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s ; CHECK-LABEL: convert_v3f32 ; CHECK: strb diff --git a/llvm/test/CodeGen/AArch64/v8.5a-neon-frint3264-intrinsic.ll b/llvm/test/CodeGen/AArch64/v8.5a-neon-frint3264-intrinsic.ll index 1979d97952cb..85187f1ee4ca 100644 --- a/llvm/test/CodeGen/AArch64/v8.5a-neon-frint3264-intrinsic.ll +++ b/llvm/test/CodeGen/AArch64/v8.5a-neon-frint3264-intrinsic.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.5a | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.5a | FileCheck %s declare <2 x float> @llvm.aarch64.neon.frint32x.v2f32(<2 x float>) declare <4 x float> @llvm.aarch64.neon.frint32x.v4f32(<4 x float>) diff --git a/llvm/test/CodeGen/AArch64/v8.5a-scalar-frint3264-intrinsic.ll b/llvm/test/CodeGen/AArch64/v8.5a-scalar-frint3264-intrinsic.ll index b4a0f3d557c6..728c13da5f1b 100644 --- a/llvm/test/CodeGen/AArch64/v8.5a-scalar-frint3264-intrinsic.ll +++ b/llvm/test/CodeGen/AArch64/v8.5a-scalar-frint3264-intrinsic.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mtriple=aarch64-eabi -mattr=+v8.5a | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -mattr=+v8.5a | FileCheck %s declare float @llvm.aarch64.frint32z.f32(float) declare double @llvm.aarch64.frint32z.f64(double) diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll index de95943de311..1164e02a16c9 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define float @add_HalfS(<2 x float> %bin.rdx) { ; CHECK-SD-LABEL: add_HalfS: diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll index aaba379ad63d..99c6808724b5 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define float @add_HalfS(<2 x float> %bin.rdx) { ; CHECK-LABEL: add_HalfS: diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll index 7b93e60e49b9..e1b21705c95f 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define float @mul_HalfS(<2 x float> %bin.rdx) { ; CHECK-SD-LABEL: mul_HalfS: diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll index 67b4ebb33824..e85384e46222 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 -; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define float @mul_HalfS(<2 x float> %bin.rdx) { ; CHECK-SD-LABEL: mul_HalfS: diff --git a/llvm/test/CodeGen/AArch64/xar.ll b/llvm/test/CodeGen/AArch64/xar.ll index c602837a6e6c..d050eaf6646d 100644 --- a/llvm/test/CodeGen/AArch64/xar.ll +++ b/llvm/test/CodeGen/AArch64/xar.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=aarch64-none-eabi -mattr=+sha3 < %s | FileCheck --check-prefix=SHA3 %s -; RUN: llc -mtriple=aarch64-none-eabi -mattr=-sha3 < %s | FileCheck --check-prefix=NOSHA3 %s +; RUN: llc -mtriple=aarch64 -mattr=+sha3 < %s | FileCheck --check-prefix=SHA3 %s +; RUN: llc -mtriple=aarch64 -mattr=-sha3 < %s | FileCheck --check-prefix=NOSHA3 %s define <2 x i64> @xar(<2 x i64> %x, <2 x i64> %y) { ; SHA3-LABEL: xar: diff --git a/llvm/test/Transforms/LICM/sink-foldable.ll b/llvm/test/Transforms/LICM/sink-foldable.ll index bf2cc77e6f47..38577a5a1256 100644 --- a/llvm/test/Transforms/LICM/sink-foldable.ll +++ b/llvm/test/Transforms/LICM/sink-foldable.ll @@ -3,7 +3,7 @@ ; RUN: opt < %s -passes=licm -S | FileCheck %s -target triple = "aarch64--linux-gnueabi" +target triple = "aarch64" define ptr @test1(i32 %j, ptr readonly %P, ptr readnone %Q) { ; CHECK-LABEL: @test1( diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll index cb9ba1b088cb..22aaa563daa5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll @@ -2,7 +2,7 @@ ; RUN: opt -S < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 | FileCheck %s --check-prefix=FORCE-VEC target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -target triple = "aarch64--linux-gnueabi" +target triple = "aarch64" ; Test integer induction variable of step 2: ; for (int i = 0; i < 1024; i+=2) { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll b/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll index c47a63020281..4f7195934a79 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=aarch64--linux-gnueabi -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s -S | FileCheck %s +; RUN: opt -mtriple=aarch64 -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s -S | FileCheck %s ; The following tests contain loops for which SCEV cannot determine the backedge ; taken count. This is because the backedge taken condition is produced by an diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll index bf9146bbad4c..78798725b5b9 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll @@ -5,7 +5,7 @@ ; REQUIRES: asserts target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -target triple = "aarch64--linux-gnueabi" +target triple = "aarch64" %i8.2 = type {i8, i8} define void @i8_factor_2(ptr %data, i64 %n) { -- GitLab From 78f2eb8d0f9cf5e8a79b64f6d64deeb487502e38 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 12 Feb 2024 18:36:31 -0800 Subject: [PATCH 002/284] [test] Replace aarch64-*-{eabi,gnueabi}{,hf} with aarch64 --- llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll | 2 +- llvm/test/CodeGen/AArch64/cond-br-tuning.ll | 2 +- llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll | 2 +- llvm/test/Transforms/CallSiteSplitting/callsite-split.ll | 2 +- llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll | 2 +- llvm/test/Transforms/CalledValuePropagation/simple-memory.ll | 2 +- llvm/test/Transforms/CalledValuePropagation/simple-select.ll | 2 +- .../test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll b/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll index c2a3acbedc8b..578038b8f4da 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-gep-opt.ll @@ -5,7 +5,7 @@ ; RUN: llc -O3 -aarch64-enable-gep-opt=true -print-after=codegenprepare -mcpu=cortex-a53 < %s 2>&1 | FileCheck --check-prefix=CHECK-UseAA %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -target triple = "aarch64-linux-gnueabi" +target triple = "aarch64" ; Following test cases test enabling SeparateConstOffsetFromGEP pass in AArch64 ; backend. If useAA() returns true, it will lower a GEP with multiple indices diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll index 1e1af3e4a498..87a315a49f3e 100644 --- a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll +++ b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll @@ -2,7 +2,7 @@ ; RUN: llc -debugify-and-strip-all-safe < %s -O3 -mtriple=aarch64 -verify-machineinstrs | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" -target triple = "aarch64-linaro-linux-gnueabi" +target triple = "aarch64" ; CMN is an alias of ADDS. diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll index ee8ce72c3132..6e57cc069358 100644 --- a/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll +++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll @@ -2,7 +2,7 @@ ; RUN: opt < %s -passes='function(callsite-splitting)' -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" -target triple = "aarch64-linaro-linux-gnueabi" +target triple = "aarch64" ;CHECK-LABEL: @test_eq_eq diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll index 78b8f5b2793e..256261d0dd11 100644 --- a/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll +++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll @@ -1,7 +1,7 @@ ; RUN: opt < %s -passes='function(callsite-splitting),cgscc(inline),function(instcombine,jump-threading)' -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" -target triple = "aarch64-linaro-linux-gnueabi" +target triple = "aarch64" %struct.bitmap = type { i32, ptr } diff --git a/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll b/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll index d9739afe3a13..150c56ca8450 100644 --- a/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll +++ b/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll @@ -1,6 +1,6 @@ ; RUN: opt -passes=called-value-propagation -S < %s | FileCheck %s -target triple = "aarch64-unknown-linux-gnueabi" +target triple = "aarch64" ; This test checks that we propagate the functions through arguments and attach diff --git a/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll b/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll index 899bed8fc500..106ed849385c 100644 --- a/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll +++ b/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll @@ -1,6 +1,6 @@ ; RUN: opt -passes=called-value-propagation -S < %s | FileCheck %s -target triple = "aarch64-unknown-linux-gnueabi" +target triple = "aarch64" @global_function = internal unnamed_addr global ptr null, align 8 @global_array = common unnamed_addr global ptr null, align 8 diff --git a/llvm/test/Transforms/CalledValuePropagation/simple-select.ll b/llvm/test/Transforms/CalledValuePropagation/simple-select.ll index e4d35b8169e8..b2ea20a9449e 100644 --- a/llvm/test/Transforms/CalledValuePropagation/simple-select.ll +++ b/llvm/test/Transforms/CalledValuePropagation/simple-select.ll @@ -1,6 +1,6 @@ ; RUN: opt -passes=called-value-propagation -S < %s | FileCheck %s -target triple = "aarch64-unknown-linux-gnueabi" +target triple = "aarch64" @global_function = internal unnamed_addr global ptr null, align 8 @global_scalar = internal unnamed_addr global i64 zeroinitializer diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll index da0b2e68313c..f74dc3e5032a 100644 --- a/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll +++ b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt -passes=consthoist -consthoist-gep -S -o - %s | FileCheck %s -target triple = "aarch64-none--musleabi" +target triple = "aarch64" %0 = type { %1, %2, [9 x i16], %6, %7 } %1 = type { i32, i32, i32, i32, i32, i32, i16, i16, i8, i8, i16, i32, i32, i16, i8, i8 } -- GitLab From 95a204cf34c410c2be5f152daec2d476f1e06d84 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Tue, 13 Feb 2024 10:48:58 +0800 Subject: [PATCH 003/284] [CodeGenPassBuilder] Remove analysis registering interfaces (#81439) PassBuilder already can do this. --- llvm/include/llvm/Passes/CodeGenPassBuilder.h | 67 +------------------ 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index fa6dbd4a4973..80bbfb75185a 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -7,8 +7,7 @@ //===----------------------------------------------------------------------===// /// \file /// -/// Interfaces for registering analysis passes, producing common pass manager -/// configurations, and parsing of pass pipelines. +/// Interfaces for producing common pass manager configurations. /// //===----------------------------------------------------------------------===// @@ -137,16 +136,6 @@ public: raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const; - void registerModuleAnalyses(ModuleAnalysisManager &) const; - void registerFunctionAnalyses(FunctionAnalysisManager &) const; - void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const; - - void registerAnalyses(MachineFunctionAnalysisManager &MFAM) const { - registerModuleAnalyses(*MFAM.MAM); - registerFunctionAnalyses(*MFAM.FAM); - registerMachineFunctionAnalyses(MFAM); - } - PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { return PIC; } @@ -239,14 +228,6 @@ protected: CGPassBuilderOption Opt; PassInstrumentationCallbacks *PIC; - /// Target override these hooks to parse target-specific analyses. - void registerTargetAnalysis(ModuleAnalysisManager &) const {} - void registerTargetAnalysis(FunctionAnalysisManager &) const {} - void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {} - std::pair getTargetPassNameFromLegacyName(StringRef) const { - return {"", false}; - } - template TMC &getTM() const { return static_cast(TM); } CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); } @@ -577,52 +558,6 @@ Error CodeGenPassBuilder::verifyStartStop( return Error::success(); } -static inline AAManager registerAAAnalyses() { - AAManager AA; - - // The order in which these are registered determines their priority when - // being queried. - - // Basic AliasAnalysis support. - // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that - // BasicAliasAnalysis wins if they disagree. This is intended to help - // support "obvious" type-punning idioms. - AA.registerFunctionAnalysis(); - AA.registerFunctionAnalysis(); - AA.registerFunctionAnalysis(); - - return AA; -} - -template -void CodeGenPassBuilder::registerModuleAnalyses( - ModuleAnalysisManager &MAM) const { -#define MODULE_ANALYSIS(NAME, CREATE_PASS) \ - MAM.registerPass([&] { return CREATE_PASS; }); -#include "MachinePassRegistry.def" - derived().registerTargetAnalysis(MAM); -} - -template -void CodeGenPassBuilder::registerFunctionAnalyses( - FunctionAnalysisManager &FAM) const { - FAM.registerPass([this] { return registerAAAnalyses(); }); - -#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ - FAM.registerPass([&] { return CREATE_PASS; }); -#include "MachinePassRegistry.def" - derived().registerTargetAnalysis(FAM); -} - -template -void CodeGenPassBuilder::registerMachineFunctionAnalyses( - MachineFunctionAnalysisManager &MFAM) const { -#define MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ - MFAM.registerPass([&] { return CREATE_PASS; }); -#include "MachinePassRegistry.def" - derived().registerTargetAnalysis(MFAM); -} - template void CodeGenPassBuilder::addISelPasses(AddIRPass &addPass) const { derived().addGlobalMergePass(addPass); -- GitLab From 542a3cb9cc89c3d6fca68374bf6c35ac6c229300 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Tue, 13 Feb 2024 11:18:13 +0800 Subject: [PATCH 004/284] [ValueTracking] Compute known FPClass from dominating condition (#80941) This patch improves `computeKnownFPClass` by using context-sensitive information from `DomConditionCache`. --- llvm/lib/Analysis/DomConditionCache.cpp | 7 +- llvm/lib/Analysis/ValueTracking.cpp | 91 +++-- .../InstCombine/fpclass-from-dom-cond.ll | 322 ++++++++++++++++++ 3 files changed, 392 insertions(+), 28 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll diff --git a/llvm/lib/Analysis/DomConditionCache.cpp b/llvm/lib/Analysis/DomConditionCache.cpp index 3dad0c2e0713..c07a8a76f111 100644 --- a/llvm/lib/Analysis/DomConditionCache.cpp +++ b/llvm/lib/Analysis/DomConditionCache.cpp @@ -43,7 +43,7 @@ static void findAffectedValues(Value *Cond, if (!Visited.insert(V).second) continue; - ICmpInst::Predicate Pred; + CmpInst::Predicate Pred; Value *A, *B; // Only recurse into and/or if it matches the top-level and/or type. if (TopLevelIsAnd ? match(V, m_LogicalAnd(m_Value(A), m_Value(B))) @@ -67,6 +67,11 @@ static void findAffectedValues(Value *Cond, if (match(A, m_Add(m_Value(X), m_ConstantInt()))) AddAffected(X); } + } else if (match(Cond, m_CombineOr(m_FCmp(Pred, m_Value(A), m_Constant()), + m_Intrinsic( + m_Value(A), m_Constant())))) { + // Handle patterns that computeKnownFPClass() support. + AddAffected(A); } } } diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 0e40a02fd4de..220ef32c2d3c 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -4225,9 +4225,53 @@ llvm::fcmpImpliesClass(CmpInst::Predicate Pred, const Function &F, Value *LHS, return fcmpImpliesClass(Pred, F, LHS, *ConstRHS, LookThroughSrc); } -static FPClassTest computeKnownFPClassFromAssumes(const Value *V, - const SimplifyQuery &Q) { - FPClassTest KnownFromAssume = fcAllFlags; +static void computeKnownFPClassFromCond(const Value *V, Value *Cond, + bool CondIsTrue, + const Instruction *CxtI, + KnownFPClass &KnownFromContext) { + CmpInst::Predicate Pred; + Value *LHS; + uint64_t ClassVal = 0; + const APFloat *CRHS; + // TODO: handle sign-bit check idiom + if (match(Cond, m_FCmp(Pred, m_Value(LHS), m_APFloat(CRHS)))) { + auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass( + Pred, *CxtI->getParent()->getParent(), LHS, *CRHS, LHS != V); + if (CmpVal == V) + KnownFromContext.knownNot(~(CondIsTrue ? MaskIfTrue : MaskIfFalse)); + } else if (match(Cond, m_Intrinsic( + m_Value(LHS), m_ConstantInt(ClassVal)))) { + FPClassTest Mask = static_cast(ClassVal); + KnownFromContext.knownNot(CondIsTrue ? ~Mask : Mask); + } +} + +static KnownFPClass computeKnownFPClassFromContext(const Value *V, + const SimplifyQuery &Q) { + KnownFPClass KnownFromContext; + + if (!Q.CxtI) + return KnownFromContext; + + if (Q.DC && Q.DT) { + // Handle dominating conditions. + for (BranchInst *BI : Q.DC->conditionsFor(V)) { + Value *Cond = BI->getCondition(); + + BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0)); + if (Q.DT->dominates(Edge0, Q.CxtI->getParent())) + computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, Q.CxtI, + KnownFromContext); + + BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1)); + if (Q.DT->dominates(Edge1, Q.CxtI->getParent())) + computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, Q.CxtI, + KnownFromContext); + } + } + + if (!Q.AC) + return KnownFromContext; // Try to restrict the floating-point classes based on information from // assumptions. @@ -4245,25 +4289,11 @@ static FPClassTest computeKnownFPClassFromAssumes(const Value *V, if (!isValidAssumeForContext(I, Q.CxtI, Q.DT)) continue; - CmpInst::Predicate Pred; - Value *LHS, *RHS; - uint64_t ClassVal = 0; - if (match(I->getArgOperand(0), m_FCmp(Pred, m_Value(LHS), m_Value(RHS)))) { - const APFloat *CRHS; - if (match(RHS, m_APFloat(CRHS))) { - auto [CmpVal, MaskIfTrue, MaskIfFalse] = - fcmpImpliesClass(Pred, *F, LHS, *CRHS, LHS != V); - if (CmpVal == V) - KnownFromAssume &= MaskIfTrue; - } - } else if (match(I->getArgOperand(0), - m_Intrinsic( - m_Value(LHS), m_ConstantInt(ClassVal)))) { - KnownFromAssume &= static_cast(ClassVal); - } + computeKnownFPClassFromCond(V, I->getArgOperand(0), /*CondIsTrue=*/true, + Q.CxtI, KnownFromContext); } - return KnownFromAssume; + return KnownFromContext; } void computeKnownFPClass(const Value *V, const APInt &DemandedElts, @@ -4371,10 +4401,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts, KnownNotFromFlags |= fcInf; } - if (Q.AC) { - FPClassTest AssumedClasses = computeKnownFPClassFromAssumes(V, Q); - KnownNotFromFlags |= ~AssumedClasses; - } + KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q); + KnownNotFromFlags |= ~AssumedClasses.KnownFPClasses; // We no longer need to find out about these bits from inputs if we can // assume this from flags/attributes. @@ -4382,6 +4410,12 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts, auto ClearClassesFromFlags = make_scope_exit([=, &Known] { Known.knownNot(KnownNotFromFlags); + if (!Known.SignBit && AssumedClasses.SignBit) { + if (*AssumedClasses.SignBit) + Known.signBitMustBeOne(); + else + Known.signBitMustBeZero(); + } }); if (!Op) @@ -5283,7 +5317,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts, bool First = true; - for (Value *IncValue : P->incoming_values()) { + for (const Use &U : P->operands()) { + Value *IncValue = U.get(); // Skip direct self references. if (IncValue == P) continue; @@ -5292,8 +5327,10 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts, // Recurse, but cap the recursion to two levels, because we don't want // to waste time spinning around in loops. We need at least depth 2 to // detect known sign bits. - computeKnownFPClass(IncValue, DemandedElts, InterestedClasses, KnownSrc, - PhiRecursionLimit, Q); + computeKnownFPClass( + IncValue, DemandedElts, InterestedClasses, KnownSrc, + PhiRecursionLimit, + Q.getWithInstruction(P->getIncomingBlock(U)->getTerminator())); if (First) { Known = KnownSrc; diff --git a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll new file mode 100644 index 000000000000..5d4840159dc9 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll @@ -0,0 +1,322 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +define i1 @test1(float %x) { +; CHECK-LABEL: define i1 @test1( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[COND:%.*]] = fcmp ueq float [[X]], 0.000000e+00 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.else: +; CHECK-NEXT: [[RET:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 780) +; CHECK-NEXT: ret i1 [[RET]] +; +entry: + %cond = fcmp ueq float %x, 0.000000e+00 + br i1 %cond, label %if.then, label %if.else + +if.then: + ret i1 false + +if.else: + %ret = call i1 @llvm.is.fpclass.f32(float %x, i32 783) + ret i1 %ret +} + +define i1 @test2(double %x) { +; CHECK-LABEL: define i1 @test2( +; CHECK-SAME: double [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[X]], 0x3EB0C6F7A0000000 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.end: +; CHECK-NEXT: ret i1 false +; +entry: + %cmp = fcmp olt double %x, 0x3EB0C6F7A0000000 + br i1 %cmp, label %if.then, label %if.end +if.then: + ret i1 false +if.end: + %cmp.i = fcmp oeq double %x, 0.000000e+00 + ret i1 %cmp.i +} + +define i1 @test3(float %x) { +; CHECK-LABEL: define i1 @test3( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[X]], 3.000000e+00 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[RET:%.*]] = fcmp oeq float [[X]], 0x7FF0000000000000 +; CHECK-NEXT: ret i1 [[RET]] +; CHECK: if.else: +; CHECK-NEXT: ret i1 false +; +entry: + %cmp = fcmp ogt float %x, 3.000000e+00 + br i1 %cmp, label %if.then, label %if.else +if.then: + %abs = call float @llvm.fabs.f32(float %x) + %ret = fcmp oeq float %abs, 0x7FF0000000000000 + ret i1 %ret +if.else: + ret i1 false +} + +define float @test4(float %x) { +; CHECK-LABEL: define float @test4( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[X]], 0x3EB0C6F7A0000000 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret float 1.000000e+00 +; CHECK: if.end: +; CHECK-NEXT: [[DIV:%.*]] = fdiv float 1.000000e+00, [[X]] +; CHECK-NEXT: ret float [[DIV]] +; +entry: + %cmp = fcmp olt float %x, 0x3EB0C6F7A0000000 + br i1 %cmp, label %if.then, label %if.end + +if.then: + ret float 1.0 + +if.end: + %cmp.i = fcmp oeq float %x, 0.000000e+00 + %div = fdiv float 1.000000e+00, %x + %ret = select i1 %cmp.i, float 1.000000e+00, float %div + ret float %ret +} + +define i1 @test5(double %x, i1 %cond) { +; CHECK-LABEL: define i1 @test5( +; CHECK-SAME: double [[X:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[CMP:%.*]] = fcmp uno double [[X]], 0.000000e+00 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.end: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: [[Y:%.*]] = phi double [ -1.000000e+00, [[ENTRY:%.*]] ], [ [[X]], [[IF_END]] ] +; CHECK-NEXT: [[RET:%.*]] = tail call i1 @llvm.is.fpclass.f64(double [[Y]], i32 408) +; CHECK-NEXT: ret i1 [[RET]] +; +entry: + br i1 %cond, label %if, label %exit +if: + %cmp = fcmp uno double %x, 0.000000e+00 + br i1 %cmp, label %if.then, label %if.end +if.then: + ret i1 false +if.end: + br label %exit +exit: + %y = phi double [ -1.000000e+00, %entry ], [ %x, %if.end ] + %ret = tail call i1 @llvm.is.fpclass.f64(double %y, i32 411) + ret i1 %ret +} + +define i1 @test6(double %x) { +; CHECK-LABEL: define i1 @test6( +; CHECK-SAME: double [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt double [[X]], 0.000000e+00 +; CHECK-NEXT: br i1 [[CMP]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]] +; CHECK: land.rhs: +; CHECK-NEXT: [[CMP_I:%.*]] = fcmp oeq double [[X]], 0x7FF0000000000000 +; CHECK-NEXT: br label [[LAND_END]] +; CHECK: land.end: +; CHECK-NEXT: [[RET:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP_I]], [[LAND_RHS]] ] +; CHECK-NEXT: ret i1 [[RET]] +; +entry: + %cmp = fcmp ogt double %x, 0.000000e+00 + br i1 %cmp, label %land.rhs, label %land.end + +land.rhs: + %abs = tail call double @llvm.fabs.f64(double %x) + %and.i = bitcast double %abs to i64 + %cmp.i = icmp eq i64 %and.i, 9218868437227405312 + br label %land.end + +land.end: + %ret = phi i1 [ false, %entry ], [ %cmp.i, %land.rhs ] + ret i1 %ret +} + +define i1 @test7(float %x) { +; CHECK-LABEL: define i1 @test7( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: [[COND:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 345) +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[RET1:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 328) +; CHECK-NEXT: ret i1 [[RET1]] +; CHECK: if.else: +; CHECK-NEXT: [[RET2:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 128) +; CHECK-NEXT: ret i1 [[RET2]] +; + %cond = call i1 @llvm.is.fpclass.f32(float %x, i32 345) + br i1 %cond, label %if.then, label %if.else +if.then: + %ret1 = call i1 @llvm.is.fpclass.f32(float %x, i32 456) + ret i1 %ret1 +if.else: + %ret2 = call i1 @llvm.is.fpclass.f32(float %x, i32 456) + ret i1 %ret2 +} + +; TODO: These two is.fpclass can be simplified. +define i1 @test8(float %x) { +; CHECK-LABEL: define i1 @test8( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: [[ABS:%.*]] = call float @llvm.fabs.f32(float [[X]]) +; CHECK-NEXT: [[COND:%.*]] = fcmp oeq float [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[RET1:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 575) +; CHECK-NEXT: ret i1 [[RET1]] +; CHECK: if.else: +; CHECK-NEXT: [[RET2:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 575) +; CHECK-NEXT: ret i1 [[RET2]] +; + %abs = call float @llvm.fabs.f32(float %x) + %cond = fcmp oeq float %abs, 0x7FF0000000000000 + br i1 %cond, label %if.then, label %if.else +if.then: + %ret1 = call i1 @llvm.is.fpclass.f32(float %x, i32 575) + ret i1 %ret1 +if.else: + %ret2 = call i1 @llvm.is.fpclass.f32(float %x, i32 575) + ret i1 %ret2 +} + +define i1 @test9(float %x) { +; CHECK-LABEL: define i1 @test9( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: [[COND:%.*]] = fcmp olt float [[X]], -1.000000e+00 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.else: +; CHECK-NEXT: ret i1 false +; + %cond = fcmp olt float %x, -1.0 + br i1 %cond, label %if.then, label %if.else +if.then: + %ret1 = fcmp oeq float %x, 0x7FF0000000000000 + ret i1 %ret1 +if.else: + ret i1 false +} + +define i1 @test10(float %x) { +; CHECK-LABEL: define i1 @test10( +; CHECK-SAME: float [[X:%.*]]) { +; CHECK-NEXT: [[COND:%.*]] = fcmp olt float [[X]], -1.000000e+00 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.else: +; CHECK-NEXT: ret i1 false +; + %cond = fcmp olt float %x, -1.0 + %neg = fneg float %x + br i1 %cond, label %if.then, label %if.else +if.then: + %ret1 = fcmp oeq float %neg, 0xFFF0000000000000 + ret i1 %ret1 +if.else: + ret i1 false +} + +; TODO: handle and/or conditions +define i1 @test11_and(float %x, i1 %cond2) { +; CHECK-LABEL: define i1 @test11_and( +; CHECK-SAME: float [[X:%.*]], i1 [[COND2:%.*]]) { +; CHECK-NEXT: [[COND:%.*]] = fcmp olt float [[X]], -1.000000e+00 +; CHECK-NEXT: [[AND:%.*]] = and i1 [[COND]], [[COND2]] +; CHECK-NEXT: br i1 [[AND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[RET1:%.*]] = fcmp oeq float [[X]], 0x7FF0000000000000 +; CHECK-NEXT: ret i1 [[RET1]] +; CHECK: if.else: +; CHECK-NEXT: ret i1 false +; + %cond = fcmp olt float %x, -1.0 + %neg = fneg float %x + %and = and i1 %cond, %cond2 + br i1 %and, label %if.then, label %if.else +if.then: + %ret1 = fcmp oeq float %neg, 0xFFF0000000000000 + ret i1 %ret1 +if.else: + ret i1 false +} + +; TODO: handle and/or conditions +define i1 @test12_or(float %x, i1 %cond2) { +; CHECK-LABEL: define i1 @test12_or( +; CHECK-SAME: float [[X:%.*]], i1 [[COND2:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[COND:%.*]] = fcmp ueq float [[X]], 0.000000e+00 +; CHECK-NEXT: [[OR:%.*]] = or i1 [[COND]], [[COND2]] +; CHECK-NEXT: br i1 [[OR]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.else: +; CHECK-NEXT: [[RET:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 783) +; CHECK-NEXT: ret i1 [[RET]] +; +entry: + %cond = fcmp ueq float %x, 0.000000e+00 + %or = or i1 %cond, %cond2 + br i1 %or, label %if.then, label %if.else + +if.then: + ret i1 false + +if.else: + %ret = call i1 @llvm.is.fpclass.f32(float %x, i32 783) + ret i1 %ret +} + +define i1 @test1_no_dominating(float %x, i1 %c) { +; CHECK-LABEL: define i1 @test1_no_dominating( +; CHECK-SAME: float [[X:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: entry0: +; CHECK-NEXT: br i1 [[C]], label [[ENTRY:%.*]], label [[IF_ELSE:%.*]] +; CHECK: entry: +; CHECK-NEXT: [[COND:%.*]] = fcmp ueq float [[X]], 0.000000e+00 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE]] +; CHECK: if.then: +; CHECK-NEXT: ret i1 false +; CHECK: if.else: +; CHECK-NEXT: [[RET:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 783) +; CHECK-NEXT: ret i1 [[RET]] +; +entry0: + br i1 %c, label %entry, label %if.else + +entry: + %cond = fcmp ueq float %x, 0.000000e+00 + br i1 %cond, label %if.then, label %if.else + +if.then: + ret i1 false + +if.else: + %ret = call i1 @llvm.is.fpclass.f32(float %x, i32 783) + ret i1 %ret +} -- GitLab From 4af24d4ab76539706bfbceec4b3923426fb1b9e7 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Mon, 12 Feb 2024 19:20:26 -0800 Subject: [PATCH 005/284] [clang-format] Don't remove parentheses in macro definitions (#81444) Closes #81399. --- clang/lib/Format/UnwrappedLineParser.cpp | 2 +- clang/unittests/Format/FormatTest.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index d84914c6a4cf..8f6453a25d9d 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -2518,7 +2518,7 @@ bool UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) { parseChildBlock(); break; case tok::r_paren: - if (!MightBeStmtExpr && + if (!MightBeStmtExpr && !Line->InMacroBody && Style.RemoveParentheses > FormatStyle::RPS_Leave) { const auto *Prev = LeftParen->Previous; const auto *Next = Tokens->peekNextToken(); diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 7b65c8d6e21b..13937a15fdae 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -26972,6 +26972,7 @@ TEST_F(FormatTest, RemoveParentheses) { EXPECT_EQ(Style.RemoveParentheses, FormatStyle::RPS_Leave); Style.RemoveParentheses = FormatStyle::RPS_MultipleParentheses; + verifyFormat("#define Foo(...) foo((__VA_ARGS__))", Style); verifyFormat("int x __attribute__((aligned(16))) = 0;", Style); verifyFormat("decltype((foo->bar)) baz;", Style); verifyFormat("class __declspec(dllimport) X {};", @@ -27006,6 +27007,7 @@ TEST_F(FormatTest, RemoveParentheses) { verifyFormat("return (({ 0; }));", "return ((({ 0; })));", Style); Style.RemoveParentheses = FormatStyle::RPS_ReturnStatement; + verifyFormat("#define Return0 return (0);", Style); verifyFormat("return 0;", "return (0);", Style); verifyFormat("co_return 0;", "co_return ((0));", Style); verifyFormat("return 0;", "return (((0)));", Style); -- GitLab From c43ad6c0fddac0bbed5e881801dd2bc2f9eeba2d Mon Sep 17 00:00:00 2001 From: yonghong-song Date: Mon, 12 Feb 2024 20:08:01 -0800 Subject: [PATCH 006/284] BPF: Change callx insn encoding (#81546) Currently, the kernel verifier unsupported callx insn used the 32-bit imm field to store the target register. On the other hand, gcc used the dst_reg field to store the target register. The gcc encoding is better. This patch adjusted the coding to be the same as gcc. Signed-off-by: Yonghong Song --- llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 1 + llvm/lib/Target/BPF/BPFInstrInfo.td | 4 ++-- llvm/test/MC/BPF/insn-unit.s | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index 90697c6645be..0d1eef60c3b5 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -229,6 +229,7 @@ public: return StringSwitch(Name.lower()) .Case("if", true) .Case("call", true) + .Case("callx", true) .Case("goto", true) .Case("gotol", true) .Case("*", true) diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 7d443a344901..690d53420718 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -622,9 +622,9 @@ class CALLX (ins GPR:$BrDst), !strconcat(OpcodeStr, " $BrDst"), []> { - bits<32> BrDst; + bits<4> BrDst; - let Inst{31-0} = BrDst; + let Inst{51-48} = BrDst; let BPFClass = BPF_JMP; } diff --git a/llvm/test/MC/BPF/insn-unit.s b/llvm/test/MC/BPF/insn-unit.s index 58342cda7cc0..224eb7381aa2 100644 --- a/llvm/test/MC/BPF/insn-unit.s +++ b/llvm/test/MC/BPF/insn-unit.s @@ -61,6 +61,9 @@ // CHECK-32: c3 92 10 00 00 00 00 00 lock *(u32 *)(r2 + 16) += w9 // CHECK: db a3 e2 ff 00 00 00 00 lock *(u64 *)(r3 - 30) += r10 + callx r2 +// CHECK: 8d 02 00 00 00 00 00 00 callx r2 + // ======== BPF_JMP Class ======== if r1 & r2 goto Llabel0 // BPF_JSET | BPF_X if r1 & 0xffff goto Llabel0 // BPF_JSET | BPF_K -- GitLab From 070fad422834faab1f0ffc90e21e25d2f524b17b Mon Sep 17 00:00:00 2001 From: Kareem Ergawy Date: Tue, 13 Feb 2024 05:10:18 +0100 Subject: [PATCH 007/284] [MLIR][OpenMP] Add `omp.private` op (#80955) This PR adds a new op to the OpenMP dialect: `PrivateClauseOp`. This op will be later used to model `[first]private` clauses for differnt OpenMP directives. This is part of productizing the "delayed privatization" PoC which can be found in #79862. --- mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 93 ++++++++++++++++++- mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp | 67 +++++++++++++ mlir/test/Dialect/OpenMP/invalid.mlir | 63 +++++++++++++ mlir/test/Dialect/OpenMP/roundtrip.mlir | 21 +++++ 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 mlir/test/Dialect/OpenMP/roundtrip.mlir diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td index 5d84217b9c70..44f3e5b8dbc3 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td @@ -133,6 +133,97 @@ def DeclareTargetAttr : OpenMP_Attr<"DeclareTarget", "declaretarget"> { let assemblyFormat = "`<` struct(params) `>`"; } +//===----------------------------------------------------------------------===// +// 2.19.4 Data-Sharing Attribute Clauses +//===----------------------------------------------------------------------===// + +def DataSharingTypePrivate : I32EnumAttrCase<"Private", 0, "private">; +def DataSharingTypeFirstPrivate : I32EnumAttrCase<"FirstPrivate", 1, "firstprivate">; + +def DataSharingClauseType : I32EnumAttr< + "DataSharingClauseType", + "Type of a data-sharing clause", + [DataSharingTypePrivate, DataSharingTypeFirstPrivate]> { + let genSpecializedAttr = 0; + let cppNamespace = "::mlir::omp"; +} + +def DataSharingClauseTypeAttr : EnumAttr< + OpenMP_Dialect, DataSharingClauseType, "data_sharing_type"> { + let assemblyFormat = "`{` `type` `=` $value `}`"; +} + +def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove]> { + let summary = "Provides declaration of [first]private logic."; + let description = [{ + This operation provides a declaration of how to implement the + [first]privatization of a variable. The dialect users should provide + information about how to create an instance of the type in the alloc region + and how to initialize the copy from the original item in the copy region. + + Examples: + --------- + * `private(x)` would be emitted as: + ```mlir + omp.private {type = private} @x.privatizer : !fir.ref alloc { + ^bb0(%arg0: !fir.ref): + %0 = ... allocate proper memory for the private clone ... + omp.yield(%0 : !fir.ref) + } + ``` + + * `firstprivate(x)` would be emitted as: + ```mlir + omp.private {type = firstprivate} @x.privatizer : !fir.ref alloc { + ^bb0(%arg0: !fir.ref): + %0 = ... allocate proper memory for the private clone ... + omp.yield(%0 : !fir.ref) + } copy { + ^bb0(%arg0: !fir.ref, %arg1: !fir.ref): + // %arg0 is the original host variable. Same as for `alloc`. + // %arg1 represents the memory allocated in `alloc`. + ... copy from host to the privatized clone .... + omp.yield(%arg1 : !fir.ref) + } + ``` + + There are no restrictions on the body except for: + - The `alloc` region has a single argument. + - The `copy` region has 2 arguments. + - Both regions are terminated by `omp.yield` ops. + The above restrictions and other obvious restrictions (e.g. verifying the + type of yielded values) are verified by the custom op verifier. The actual + contents of the blocks inside both regions are not verified. + + Instances of this op would then be used by ops that model directives that + accept data-sharing attribute clauses. + + The $sym_name attribute provides a symbol by which the privatizer op can be + referenced by other dialect ops. + + The $type attribute is the type of the value being privatized. + + The $data_sharing_type attribute specifies whether privatizer corresponds + to a `private` or a `firstprivate` clause. + }]; + + let arguments = (ins SymbolNameAttr:$sym_name, + TypeAttrOf:$type, + DataSharingClauseTypeAttr:$data_sharing_type); + + let regions = (region MinSizedRegion<1>:$alloc_region, + AnyRegion:$copy_region); + + let assemblyFormat = [{ + $data_sharing_type $sym_name `:` $type + `alloc` $alloc_region + (`copy` $copy_region^)? + attr-dict + }]; + + let hasVerifier = 1; +} + //===----------------------------------------------------------------------===// // 2.6 parallel Construct //===----------------------------------------------------------------------===// @@ -609,7 +700,7 @@ def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments, def YieldOp : OpenMP_Op<"yield", [Pure, ReturnLike, Terminator, ParentOneOf<["WsLoopOp", "ReductionDeclareOp", - "AtomicUpdateOp", "SimdLoopOp"]>]> { + "AtomicUpdateOp", "SimdLoopOp", "PrivateClauseOp"]>]> { let summary = "loop yield and termination operation"; let description = [{ "omp.yield" yields SSA values from the OpenMP dialect op region and diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index 394f0629ba6e..ef08bd87efc9 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -1662,6 +1662,73 @@ LogicalResult DataBoundsOp::verify() { return success(); } +LogicalResult PrivateClauseOp::verify() { + Type symType = getType(); + + auto verifyTerminator = [&](Operation *terminator) -> LogicalResult { + if (!terminator->hasSuccessors() && !llvm::isa(terminator)) + return mlir::emitError(terminator->getLoc()) + << "expected exit block terminator to be an `omp.yield` op."; + + YieldOp yieldOp = llvm::cast(terminator); + TypeRange yieldedTypes = yieldOp.getResults().getTypes(); + + if (yieldedTypes.size() == 1 && yieldedTypes.front() == symType) + return success(); + + auto error = mlir::emitError(yieldOp.getLoc()) + << "Invalid yielded value. Expected type: " << symType + << ", got: "; + + if (yieldedTypes.empty()) + error << "None"; + else + error << yieldedTypes; + + return error; + }; + + auto verifyRegion = [&](Region ®ion, unsigned expectedNumArgs, + StringRef regionName) -> LogicalResult { + assert(!region.empty()); + + if (region.getNumArguments() != expectedNumArgs) + return mlir::emitError(region.getLoc()) + << "`" << regionName << "`: " + << "expected " << expectedNumArgs + << " region arguments, got: " << region.getNumArguments(); + + for (Block &block : region) { + // MLIR will verify the absence of the terminator for us. + if (!block.mightHaveTerminator()) + continue; + + if (failed(verifyTerminator(block.getTerminator()))) + return failure(); + } + + return success(); + }; + + if (failed(verifyRegion(getAllocRegion(), /*expectedNumArgs=*/1, "alloc"))) + return failure(); + + DataSharingClauseType dsType = getDataSharingType(); + + if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty()) + return emitError("`private` clauses require only an `alloc` region."); + + if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty()) + return emitError( + "`firstprivate` clauses require both `alloc` and `copy` regions."); + + if (dsType == DataSharingClauseType::FirstPrivate && + failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy"))) + return failure(); + + return success(); +} + #define GET_ATTRDEF_CLASSES #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc" diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir index 812b79e35595..59b42390b206 100644 --- a/mlir/test/Dialect/OpenMP/invalid.mlir +++ b/mlir/test/Dialect/OpenMP/invalid.mlir @@ -1738,3 +1738,66 @@ func.func @omp_distribute(%data_var : memref) -> () { "omp.terminator"() : () -> () }) : (memref) -> () } + +// ----- + +omp.private {type = private} @x.privatizer : i32 alloc { +^bb0(%arg0: i32): + %0 = arith.constant 0.0 : f32 + // expected-error @below {{Invalid yielded value. Expected type: 'i32', got: 'f32'}} + omp.yield(%0 : f32) +} + +// ----- + +omp.private {type = private} @x.privatizer : i32 alloc { +^bb0(%arg0: i32): + // expected-error @below {{Invalid yielded value. Expected type: 'i32', got: None}} + omp.yield +} + +// ----- + +omp.private {type = private} @x.privatizer : i32 alloc { +^bb0(%arg0: i32): + // expected-error @below {{expected exit block terminator to be an `omp.yield` op.}} + omp.terminator +} + +// ----- + +// expected-error @below {{`alloc`: expected 1 region arguments, got: 2}} +omp.private {type = private} @x.privatizer : f32 alloc { +^bb0(%arg0: f32, %arg1: f32): + omp.yield(%arg0 : f32) +} + +// ----- + +// expected-error @below {{`copy`: expected 2 region arguments, got: 1}} +omp.private {type = firstprivate} @x.privatizer : f32 alloc { +^bb0(%arg0: f32): + omp.yield(%arg0 : f32) +} copy { +^bb0(%arg0: f32): + omp.yield(%arg0 : f32) +} + +// ----- + +// expected-error @below {{`private` clauses require only an `alloc` region.}} +omp.private {type = private} @x.privatizer : f32 alloc { +^bb0(%arg0: f32): + omp.yield(%arg0 : f32) +} copy { +^bb0(%arg0: f32, %arg1 : f32): + omp.yield(%arg0 : f32) +} + +// ----- + +// expected-error @below {{`firstprivate` clauses require both `alloc` and `copy` regions.}} +omp.private {type = firstprivate} @x.privatizer : f32 alloc { +^bb0(%arg0: f32): + omp.yield(%arg0 : f32) +} diff --git a/mlir/test/Dialect/OpenMP/roundtrip.mlir b/mlir/test/Dialect/OpenMP/roundtrip.mlir new file mode 100644 index 000000000000..2553442638ee --- /dev/null +++ b/mlir/test/Dialect/OpenMP/roundtrip.mlir @@ -0,0 +1,21 @@ +// RUN: mlir-opt -verify-diagnostics %s | mlir-opt | FileCheck %s + +// CHECK: omp.private {type = private} @x.privatizer : !llvm.ptr alloc { +omp.private {type = private} @x.privatizer : !llvm.ptr alloc { +// CHECK: ^bb0(%arg0: {{.*}}): +^bb0(%arg0: !llvm.ptr): + omp.yield(%arg0 : !llvm.ptr) +} + +// CHECK: omp.private {type = firstprivate} @y.privatizer : !llvm.ptr alloc { +omp.private {type = firstprivate} @y.privatizer : !llvm.ptr alloc { +// CHECK: ^bb0(%arg0: {{.*}}): +^bb0(%arg0: !llvm.ptr): + omp.yield(%arg0 : !llvm.ptr) +// CHECK: } copy { +} copy { +// CHECK: ^bb0(%arg0: {{.*}}, %arg1: {{.*}}): +^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr): + omp.yield(%arg0 : !llvm.ptr) +} + -- GitLab From 91dcf53abd34fa836a126c706f87b810d299d802 Mon Sep 17 00:00:00 2001 From: rmarker <37921131+rmarker@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:58:33 +1030 Subject: [PATCH 008/284] [clang-format] Rename option AlwaysBreakAfterReturnType. (#80827) Changes the option to BreakAfterReturnType option, with a more relevant name, deprecating and replacing AlwaysBreakAfterReturnType. Following up on #78010. --- clang/docs/ClangFormatStyleOptions.rst | 221 +++++++++++---------- clang/docs/ReleaseNotes.rst | 2 + clang/include/clang/Format/Format.h | 7 +- clang/lib/Format/Format.cpp | 5 +- clang/unittests/Format/ConfigParseTest.cpp | 16 ++ 5 files changed, 140 insertions(+), 111 deletions(-) diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst index 5deeff0db239..fdf7bfaeaa4e 100644 --- a/clang/docs/ClangFormatStyleOptions.rst +++ b/clang/docs/ClangFormatStyleOptions.rst @@ -1531,114 +1531,8 @@ the configuration (without a prefix: ``Auto``). .. _AlwaysBreakAfterReturnType: -**AlwaysBreakAfterReturnType** (``ReturnTypeBreakingStyle``) :versionbadge:`clang-format 3.8` :ref:`¶ ` - The function declaration return type breaking style to use. - - Possible values: - - * ``RTBS_None`` (in configuration: ``None``) - This is **deprecated**. See ``Automatic`` below. - - * ``RTBS_Automatic`` (in configuration: ``Automatic``) - Break after return type based on ``PenaltyReturnTypeOnItsOwnLine``. - - .. code-block:: c++ - - class A { - int f() { return 0; }; - }; - int f(); - int f() { return 1; } - int - LongName::AnotherLongName(); - - * ``RTBS_ExceptShortType`` (in configuration: ``ExceptShortType``) - Same as ``Automatic`` above, except that there is no break after short - return types. - - .. code-block:: c++ - - class A { - int f() { return 0; }; - }; - int f(); - int f() { return 1; } - int LongName:: - AnotherLongName(); - - * ``RTBS_All`` (in configuration: ``All``) - Always break after the return type. - - .. code-block:: c++ - - class A { - int - f() { - return 0; - }; - }; - int - f(); - int - f() { - return 1; - } - int - LongName::AnotherLongName(); - - * ``RTBS_TopLevel`` (in configuration: ``TopLevel``) - Always break after the return types of top-level functions. - - .. code-block:: c++ - - class A { - int f() { return 0; }; - }; - int - f(); - int - f() { - return 1; - } - int - LongName::AnotherLongName(); - - * ``RTBS_AllDefinitions`` (in configuration: ``AllDefinitions``) - Always break after the return type of function definitions. - - .. code-block:: c++ - - class A { - int - f() { - return 0; - }; - }; - int f(); - int - f() { - return 1; - } - int - LongName::AnotherLongName(); - - * ``RTBS_TopLevelDefinitions`` (in configuration: ``TopLevelDefinitions``) - Always break after the return type of top-level definitions. - - .. code-block:: c++ - - class A { - int f() { return 0; }; - }; - int f(); - int - f() { - return 1; - } - int - LongName::AnotherLongName(); - - +**AlwaysBreakAfterReturnType** (``deprecated``) :versionbadge:`clang-format 3.8` :ref:`¶ ` + This option is renamed to ``BreakAfterReturnType``. .. _AlwaysBreakBeforeMultilineStrings: @@ -2219,6 +2113,117 @@ the configuration (without a prefix: ``Auto``). @Mock DataLoad loader; +.. _BreakAfterReturnType: + +**BreakAfterReturnType** (``ReturnTypeBreakingStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` + The function declaration return type breaking style to use. + + Possible values: + + * ``RTBS_None`` (in configuration: ``None``) + This is **deprecated**. See ``Automatic`` below. + + * ``RTBS_Automatic`` (in configuration: ``Automatic``) + Break after return type based on ``PenaltyReturnTypeOnItsOwnLine``. + + .. code-block:: c++ + + class A { + int f() { return 0; }; + }; + int f(); + int f() { return 1; } + int + LongName::AnotherLongName(); + + * ``RTBS_ExceptShortType`` (in configuration: ``ExceptShortType``) + Same as ``Automatic`` above, except that there is no break after short + return types. + + .. code-block:: c++ + + class A { + int f() { return 0; }; + }; + int f(); + int f() { return 1; } + int LongName:: + AnotherLongName(); + + * ``RTBS_All`` (in configuration: ``All``) + Always break after the return type. + + .. code-block:: c++ + + class A { + int + f() { + return 0; + }; + }; + int + f(); + int + f() { + return 1; + } + int + LongName::AnotherLongName(); + + * ``RTBS_TopLevel`` (in configuration: ``TopLevel``) + Always break after the return types of top-level functions. + + .. code-block:: c++ + + class A { + int f() { return 0; }; + }; + int + f(); + int + f() { + return 1; + } + int + LongName::AnotherLongName(); + + * ``RTBS_AllDefinitions`` (in configuration: ``AllDefinitions``) + Always break after the return type of function definitions. + + .. code-block:: c++ + + class A { + int + f() { + return 0; + }; + }; + int f(); + int + f() { + return 1; + } + int + LongName::AnotherLongName(); + + * ``RTBS_TopLevelDefinitions`` (in configuration: ``TopLevelDefinitions``) + Always break after the return type of top-level definitions. + + .. code-block:: c++ + + class A { + int f() { return 0; }; + }; + int f(); + int + f() { + return 1; + } + int + LongName::AnotherLongName(); + + + .. _BreakArrays: **BreakArrays** (``Boolean``) :versionbadge:`clang-format 16` :ref:`¶ ` diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 402a2f868738..aa167d75c3bf 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -296,6 +296,8 @@ clang-format - ``AlwaysBreakTemplateDeclarations`` is deprecated and renamed to ``BreakTemplateDeclarations``. +- ``AlwaysBreakAfterReturnType`` is deprecated and renamed to + ``BreakAfterReturnType``. libclang -------- diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index d9c18e5ec1dc..737cbfced9e9 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -1010,8 +1010,9 @@ struct FormatStyle { /// \version 3.7 DefinitionReturnTypeBreakingStyle AlwaysBreakAfterDefinitionReturnType; - /// The function declaration return type breaking style to use. + /// This option is renamed to ``BreakAfterReturnType``. /// \version 3.8 + /// @deprecated ReturnTypeBreakingStyle AlwaysBreakAfterReturnType; /// If ``true``, always break before multiline string literals. @@ -1576,6 +1577,10 @@ struct FormatStyle { /// \version 16 AttributeBreakingStyle BreakAfterAttributes; + /// The function declaration return type breaking style to use. + /// \version 19 + // ReturnTypeBreakingStyle BreakAfterReturnType; + /// If ``true``, clang-format will always break after a Json array ``[`` /// otherwise it will scan until the closing ``]`` to determine if it should /// add newlines between elements (prettier compatible). diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index d2cc466744ac..8efc42e0576c 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -877,6 +877,8 @@ template <> struct MappingTraits { if (!IO.outputting()) { IO.mapOptional("AlignEscapedNewlinesLeft", Style.AlignEscapedNewlines); IO.mapOptional("AllowAllConstructorInitializersOnNextLine", OnNextLine); + IO.mapOptional("AlwaysBreakAfterReturnType", + Style.AlwaysBreakAfterReturnType); IO.mapOptional("AlwaysBreakTemplateDeclarations", Style.BreakTemplateDeclarations); IO.mapOptional("BreakBeforeInheritanceComma", @@ -941,8 +943,6 @@ template <> struct MappingTraits { Style.AllowShortLoopsOnASingleLine); IO.mapOptional("AlwaysBreakAfterDefinitionReturnType", Style.AlwaysBreakAfterDefinitionReturnType); - IO.mapOptional("AlwaysBreakAfterReturnType", - Style.AlwaysBreakAfterReturnType); IO.mapOptional("AlwaysBreakBeforeMultilineStrings", Style.AlwaysBreakBeforeMultilineStrings); IO.mapOptional("AttributeMacros", Style.AttributeMacros); @@ -957,6 +957,7 @@ template <> struct MappingTraits { IO.mapOptional("BreakAfterAttributes", Style.BreakAfterAttributes); IO.mapOptional("BreakAfterJavaFieldAnnotations", Style.BreakAfterJavaFieldAnnotations); + IO.mapOptional("BreakAfterReturnType", Style.AlwaysBreakAfterReturnType); IO.mapOptional("BreakArrays", Style.BreakArrays); IO.mapOptional("BreakBeforeBinaryOperators", Style.BreakBeforeBinaryOperators); diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp index 571e1ebda14b..ee8a55680753 100644 --- a/clang/unittests/Format/ConfigParseTest.cpp +++ b/clang/unittests/Format/ConfigParseTest.cpp @@ -678,6 +678,22 @@ TEST(ConfigParseTest, ParsesConfiguration) { BraceWrapping.AfterControlStatement, FormatStyle::BWACS_Never); Style.AlwaysBreakAfterReturnType = FormatStyle::RTBS_All; + CHECK_PARSE("BreakAfterReturnType: None", AlwaysBreakAfterReturnType, + FormatStyle::RTBS_None); + CHECK_PARSE("BreakAfterReturnType: Automatic", AlwaysBreakAfterReturnType, + FormatStyle::RTBS_Automatic); + CHECK_PARSE("BreakAfterReturnType: ExceptShortType", + AlwaysBreakAfterReturnType, FormatStyle::RTBS_ExceptShortType); + CHECK_PARSE("BreakAfterReturnType: All", AlwaysBreakAfterReturnType, + FormatStyle::RTBS_All); + CHECK_PARSE("BreakAfterReturnType: TopLevel", AlwaysBreakAfterReturnType, + FormatStyle::RTBS_TopLevel); + CHECK_PARSE("BreakAfterReturnType: AllDefinitions", + AlwaysBreakAfterReturnType, FormatStyle::RTBS_AllDefinitions); + CHECK_PARSE("BreakAfterReturnType: TopLevelDefinitions", + AlwaysBreakAfterReturnType, + FormatStyle::RTBS_TopLevelDefinitions); + // For backward compatibility: CHECK_PARSE("AlwaysBreakAfterReturnType: None", AlwaysBreakAfterReturnType, FormatStyle::RTBS_None); CHECK_PARSE("AlwaysBreakAfterReturnType: Automatic", -- GitLab From 2dd52046816ac706a25d588efac77705793d8778 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Mon, 12 Feb 2024 20:28:09 -0800 Subject: [PATCH 009/284] Recommit "[LICM] Support integer mul/add in hoistFPAssociation. (#67736)" With a fix for build bot failure. I was accessing the type of a deleted Instruction. Original message: The reassociation this is trying to repair can happen for integer types too. This patch adds support for integer mul/add to hoistFPAssociation. The function has been renamed to hoistMulAddAssociation. I've used separate statistics and limits for integer to allow tuning flexibility. --- llvm/lib/Transforms/Scalar/LICM.cpp | 69 +++- .../Transforms/LICM/expr-reassociate-int.ll | 364 ++++++++++++++++++ 2 files changed, 414 insertions(+), 19 deletions(-) create mode 100644 llvm/test/Transforms/LICM/expr-reassociate-int.ll diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index f3e40a5cb809..9ec9c31e48e0 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -110,6 +110,9 @@ STATISTIC(NumAddSubHoisted, "Number of add/subtract expressions reassociated " "and hoisted out of the loop"); STATISTIC(NumFPAssociationsHoisted, "Number of invariant FP expressions " "reassociated and hoisted out of the loop"); +STATISTIC(NumIntAssociationsHoisted, + "Number of invariant int expressions " + "reassociated and hoisted out of the loop"); /// Memory promotion is enabled by default. static cl::opt @@ -135,6 +138,12 @@ static cl::opt FPAssociationUpperLimit( "Set upper limit for the number of transformations performed " "during a single round of hoisting the reassociated expressions.")); +cl::opt IntAssociationUpperLimit( + "licm-max-num-int-reassociations", cl::init(5U), cl::Hidden, + cl::desc( + "Set upper limit for the number of transformations performed " + "during a single round of hoisting the reassociated expressions.")); + // Experimental option to allow imprecision in LICM in pathological cases, in // exchange for faster compile. This is to be removed if MemorySSA starts to // address the same issue. LICM calls MemorySSAWalker's @@ -2661,21 +2670,29 @@ static bool hoistAddSub(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo, return false; } +static bool isReassociableOp(Instruction *I, unsigned IntOpcode, + unsigned FPOpcode) { + if (I->getOpcode() == IntOpcode) + return true; + if (I->getOpcode() == FPOpcode && I->hasAllowReassoc() && + I->hasNoSignedZeros()) + return true; + return false; +} + /// Try to reassociate expressions like ((A1 * B1) + (A2 * B2) + ...) * C where /// A1, A2, ... and C are loop invariants into expressions like /// ((A1 * C * B1) + (A2 * C * B2) + ...) and hoist the (A1 * C), (A2 * C), ... /// invariant expressions. This functions returns true only if any hoisting has /// actually occured. -static bool hoistFPAssociation(Instruction &I, Loop &L, - ICFLoopSafetyInfo &SafetyInfo, - MemorySSAUpdater &MSSAU, AssumptionCache *AC, - DominatorTree *DT) { - using namespace PatternMatch; - Value *VariantOp = nullptr, *InvariantOp = nullptr; - - if (!match(&I, m_FMul(m_Value(VariantOp), m_Value(InvariantOp))) || - !I.hasAllowReassoc() || !I.hasNoSignedZeros()) +static bool hoistMulAddAssociation(Instruction &I, Loop &L, + ICFLoopSafetyInfo &SafetyInfo, + MemorySSAUpdater &MSSAU, AssumptionCache *AC, + DominatorTree *DT) { + if (!isReassociableOp(&I, Instruction::Mul, Instruction::FMul)) return false; + Value *VariantOp = I.getOperand(0); + Value *InvariantOp = I.getOperand(1); if (L.isLoopInvariant(VariantOp)) std::swap(VariantOp, InvariantOp); if (L.isLoopInvariant(VariantOp) || !L.isLoopInvariant(InvariantOp)) @@ -2689,15 +2706,17 @@ static bool hoistFPAssociation(Instruction &I, Loop &L, Worklist.push_back(VariantBinOp); while (!Worklist.empty()) { BinaryOperator *BO = Worklist.pop_back_val(); - if (!BO->hasOneUse() || !BO->hasAllowReassoc() || !BO->hasNoSignedZeros()) + if (!BO->hasOneUse()) return false; - BinaryOperator *Op0, *Op1; - if (match(BO, m_FAdd(m_BinOp(Op0), m_BinOp(Op1)))) { - Worklist.push_back(Op0); - Worklist.push_back(Op1); + if (isReassociableOp(BO, Instruction::Add, Instruction::FAdd) && + isa(BO->getOperand(0)) && + isa(BO->getOperand(1))) { + Worklist.push_back(cast(BO->getOperand(0))); + Worklist.push_back(cast(BO->getOperand(1))); continue; } - if (BO->getOpcode() != Instruction::FMul || L.isLoopInvariant(BO)) + if (!isReassociableOp(BO, Instruction::Mul, Instruction::FMul) || + L.isLoopInvariant(BO)) return false; Use &U0 = BO->getOperandUse(0); Use &U1 = BO->getOperandUse(1); @@ -2707,7 +2726,10 @@ static bool hoistFPAssociation(Instruction &I, Loop &L, Changes.push_back(&U1); else return false; - if (Changes.size() > FPAssociationUpperLimit) + unsigned Limit = I.getType()->isIntOrIntVectorTy() + ? IntAssociationUpperLimit + : FPAssociationUpperLimit; + if (Changes.size() > Limit) return false; } if (Changes.empty()) @@ -2720,7 +2742,12 @@ static bool hoistFPAssociation(Instruction &I, Loop &L, for (auto *U : Changes) { assert(L.isLoopInvariant(U->get())); Instruction *Ins = cast(U->getUser()); - U->set(Builder.CreateFMulFMF(U->get(), Factor, Ins, "factor.op.fmul")); + Value *Mul; + if (I.getType()->isIntOrIntVectorTy()) + Mul = Builder.CreateMul(U->get(), Factor, "factor.op.mul"); + else + Mul = Builder.CreateFMulFMF(U->get(), Factor, Ins, "factor.op.fmul"); + U->set(Mul); } I.replaceAllUsesWith(VariantOp); eraseInstruction(I, SafetyInfo, MSSAU); @@ -2754,9 +2781,13 @@ static bool hoistArithmetics(Instruction &I, Loop &L, return true; } - if (hoistFPAssociation(I, L, SafetyInfo, MSSAU, AC, DT)) { + bool IsInt = I.getType()->isIntOrIntVectorTy(); + if (hoistMulAddAssociation(I, L, SafetyInfo, MSSAU, AC, DT)) { ++NumHoisted; - ++NumFPAssociationsHoisted; + if (IsInt) + ++NumIntAssociationsHoisted; + else + ++NumFPAssociationsHoisted; return true; } diff --git a/llvm/test/Transforms/LICM/expr-reassociate-int.ll b/llvm/test/Transforms/LICM/expr-reassociate-int.ll new file mode 100644 index 000000000000..63548974fb31 --- /dev/null +++ b/llvm/test/Transforms/LICM/expr-reassociate-int.ll @@ -0,0 +1,364 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -passes='loop-mssa(licm)' -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CONSTRAINED +; RUN: opt -passes='loop-mssa(licm)' -licm-max-num-int-reassociations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,CONSTRAINED + +; +; A simple loop: +; +; int j; +; +; for (j = 0; j <= i; j++) +; cells[j] = d1 * cells[j + 1] * delta; +; +; ...should be transformed by the LICM pass into this: +; +; int j; +; const uint64_t d1d = d1 * delta; +; +; for (j = 0; j <= i; j++) +; cells[j] = d1d * cells[j + 1]; +; + +define void @innermost_loop_1d_shouldhoist(i32 %i, i64 %d1, i64 %delta, ptr %cells) { +; CHECK-LABEL: define void @innermost_loop_1d_shouldhoist +; CHECK-SAME: (i32 [[I:%.*]], i64 [[D1:%.*]], i64 [[DELTA:%.*]], ptr [[CELLS:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[DELTA]], [[D1]] +; CHECK-NEXT: br label [[FOR_COND:%.*]] +; CHECK: for.cond: +; CHECK-NEXT: [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_J_1:%.*]], [[FOR_BODY:%.*]] ] +; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp sgt i32 [[J]], [[I]] +; CHECK-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; CHECK: for.body: +; CHECK-NEXT: [[ADD_J_1]] = add nuw nsw i32 [[J]], 1 +; CHECK-NEXT: [[IDXPROM_J_1:%.*]] = zext i32 [[ADD_J_1]] to i64 +; CHECK-NEXT: [[ARRAYIDX_J_1:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_1]] +; CHECK-NEXT: [[CELL_1:%.*]] = load i64, ptr [[ARRAYIDX_J_1]], align 8 +; CHECK-NEXT: [[MUL_2:%.*]] = mul i64 [[MUL_1]], [[CELL_1]] +; CHECK-NEXT: [[IDXPROM_J:%.*]] = zext i32 [[J]] to i64 +; CHECK-NEXT: [[ARRAYIDX_J:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J]] +; CHECK-NEXT: store i64 [[MUL_2]], ptr [[ARRAYIDX_J]], align 8 +; CHECK-NEXT: br label [[FOR_COND]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + br label %for.cond + +for.cond: + %j = phi i32 [ 0, %entry ], [ %add.j.1, %for.body ] + %cmp.not = icmp sgt i32 %j, %i + br i1 %cmp.not, label %for.end, label %for.body + +for.body: + %add.j.1 = add nuw nsw i32 %j, 1 + %idxprom.j.1 = zext i32 %add.j.1 to i64 + %arrayidx.j.1 = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j.1 + %cell.1 = load i64, ptr %arrayidx.j.1, align 8 + %mul.1 = mul i64 %delta, %d1 + %mul.2 = mul i64 %mul.1, %cell.1 + %idxprom.j = zext i32 %j to i64 + %arrayidx.j = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j + store i64 %mul.2, ptr %arrayidx.j, align 8 + br label %for.cond + +for.end: + ret void +} + +; +; The following loop will be modified by the 'Reassociate expressions' pass, +; +; int j; +; const uint64_t d1d = d1 * delta; +; const uint64_t d2d = d2 * delta; +; +; for (j = 0; j <= i; j++) +; cells[j] = d1d * cells[j + 1] + d2d * cells[j]; +; +; ...into this: +; +; int j; +; +; for (j = 0; j <= i; j++) +; cells[j] = (d1 * cells[j + 1] + d2 * cells[j]) * delta; +; +; We expect the LICM pass to undo this transformation. +; + +define void @innermost_loop_2d(i32 %i, i64 %d1, i64 %d2, i64 %delta, ptr %cells) { +; NOT_CONSTRAINED-LABEL: define void @innermost_loop_2d +; NOT_CONSTRAINED-SAME: (i32 [[I:%.*]], i64 [[D1:%.*]], i64 [[D2:%.*]], i64 [[DELTA:%.*]], ptr [[CELLS:%.*]]) { +; NOT_CONSTRAINED-NEXT: entry: +; NOT_CONSTRAINED-NEXT: [[FACTOR_OP_MUL:%.*]] = mul i64 [[D1]], [[DELTA]] +; NOT_CONSTRAINED-NEXT: [[FACTOR_OP_MUL1:%.*]] = mul i64 [[D2]], [[DELTA]] +; NOT_CONSTRAINED-NEXT: br label [[FOR_COND:%.*]] +; NOT_CONSTRAINED: for.cond: +; NOT_CONSTRAINED-NEXT: [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_J_1:%.*]], [[FOR_BODY:%.*]] ] +; NOT_CONSTRAINED-NEXT: [[CMP_NOT:%.*]] = icmp sgt i32 [[J]], [[I]] +; NOT_CONSTRAINED-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; NOT_CONSTRAINED: for.body: +; NOT_CONSTRAINED-NEXT: [[ADD_J_1]] = add nuw nsw i32 [[J]], 1 +; NOT_CONSTRAINED-NEXT: [[IDXPROM_J_1:%.*]] = zext i32 [[ADD_J_1]] to i64 +; NOT_CONSTRAINED-NEXT: [[ARRAYIDX_J_1:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_1]] +; NOT_CONSTRAINED-NEXT: [[CELL_1:%.*]] = load i64, ptr [[ARRAYIDX_J_1]], align 8 +; NOT_CONSTRAINED-NEXT: [[MUL_1:%.*]] = mul i64 [[CELL_1]], [[FACTOR_OP_MUL]] +; NOT_CONSTRAINED-NEXT: [[IDXPROM_J:%.*]] = zext i32 [[J]] to i64 +; NOT_CONSTRAINED-NEXT: [[ARRAYIDX_J:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J]] +; NOT_CONSTRAINED-NEXT: [[CELL_2:%.*]] = load i64, ptr [[ARRAYIDX_J]], align 8 +; NOT_CONSTRAINED-NEXT: [[MUL_2:%.*]] = mul i64 [[CELL_2]], [[FACTOR_OP_MUL1]] +; NOT_CONSTRAINED-NEXT: [[REASS_ADD:%.*]] = add i64 [[MUL_2]], [[MUL_1]] +; NOT_CONSTRAINED-NEXT: store i64 [[REASS_ADD]], ptr [[ARRAYIDX_J]], align 8 +; NOT_CONSTRAINED-NEXT: br label [[FOR_COND]] +; NOT_CONSTRAINED: for.end: +; NOT_CONSTRAINED-NEXT: ret void +; +; CONSTRAINED-LABEL: define void @innermost_loop_2d +; CONSTRAINED-SAME: (i32 [[I:%.*]], i64 [[D1:%.*]], i64 [[D2:%.*]], i64 [[DELTA:%.*]], ptr [[CELLS:%.*]]) { +; CONSTRAINED-NEXT: entry: +; CONSTRAINED-NEXT: br label [[FOR_COND:%.*]] +; CONSTRAINED: for.cond: +; CONSTRAINED-NEXT: [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_J_1:%.*]], [[FOR_BODY:%.*]] ] +; CONSTRAINED-NEXT: [[CMP_NOT:%.*]] = icmp sgt i32 [[J]], [[I]] +; CONSTRAINED-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; CONSTRAINED: for.body: +; CONSTRAINED-NEXT: [[ADD_J_1]] = add nuw nsw i32 [[J]], 1 +; CONSTRAINED-NEXT: [[IDXPROM_J_1:%.*]] = zext i32 [[ADD_J_1]] to i64 +; CONSTRAINED-NEXT: [[ARRAYIDX_J_1:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_1]] +; CONSTRAINED-NEXT: [[CELL_1:%.*]] = load i64, ptr [[ARRAYIDX_J_1]], align 8 +; CONSTRAINED-NEXT: [[MUL_1:%.*]] = mul i64 [[CELL_1]], [[D1]] +; CONSTRAINED-NEXT: [[IDXPROM_J:%.*]] = zext i32 [[J]] to i64 +; CONSTRAINED-NEXT: [[ARRAYIDX_J:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J]] +; CONSTRAINED-NEXT: [[CELL_2:%.*]] = load i64, ptr [[ARRAYIDX_J]], align 8 +; CONSTRAINED-NEXT: [[MUL_2:%.*]] = mul i64 [[CELL_2]], [[D2]] +; CONSTRAINED-NEXT: [[REASS_ADD:%.*]] = add i64 [[MUL_2]], [[MUL_1]] +; CONSTRAINED-NEXT: [[REASS_MUL:%.*]] = mul i64 [[REASS_ADD]], [[DELTA]] +; CONSTRAINED-NEXT: store i64 [[REASS_MUL]], ptr [[ARRAYIDX_J]], align 8 +; CONSTRAINED-NEXT: br label [[FOR_COND]] +; CONSTRAINED: for.end: +; CONSTRAINED-NEXT: ret void +; +entry: + br label %for.cond + +for.cond: + %j = phi i32 [ 0, %entry ], [ %add.j.1, %for.body ] + %cmp.not = icmp sgt i32 %j, %i + br i1 %cmp.not, label %for.end, label %for.body + +for.body: + %add.j.1 = add nuw nsw i32 %j, 1 + %idxprom.j.1 = zext i32 %add.j.1 to i64 + %arrayidx.j.1 = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j.1 + %cell.1 = load i64, ptr %arrayidx.j.1, align 8 + %mul.1 = mul i64 %cell.1, %d1 + %idxprom.j = zext i32 %j to i64 + %arrayidx.j = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j + %cell.2 = load i64, ptr %arrayidx.j, align 8 + %mul.2 = mul i64 %cell.2, %d2 + %reass.add = add i64 %mul.2, %mul.1 + %reass.mul = mul i64 %reass.add, %delta + store i64 %reass.mul, ptr %arrayidx.j, align 8 + br label %for.cond + +for.end: + ret void +} + +; +; The following loop will be modified by the 'Reassociate expressions' pass, +; +; int j; +; const uint64_t d1d = d1 * delta; +; const uint64_t d2d = d2 * delta; +; const uint64_t d3d = d3 * delta; +; +; for (j = 0; j <= i; j++) +; cells[j] = d1d * cells[j + 1] + d2d * cells[j] + d3d * cells[j + 2]; +; +; ...into this: +; +; int j; +; +; for (j = 0; j <= i; j++) +; cells[j] = (d1 * cells[j + 1] + d2 * cells[j] + d3 * cells[j + 2]) * delta; +; +; We expect the LICM pass to undo this transformation. +; + + +define void @innermost_loop_3d(i32 %i, i64 %d1, i64 %d2, i64 %d3, i64 %delta, ptr %cells) { +; NOT_CONSTRAINED-LABEL: define void @innermost_loop_3d +; NOT_CONSTRAINED-SAME: (i32 [[I:%.*]], i64 [[D1:%.*]], i64 [[D2:%.*]], i64 [[D3:%.*]], i64 [[DELTA:%.*]], ptr [[CELLS:%.*]]) { +; NOT_CONSTRAINED-NEXT: entry: +; NOT_CONSTRAINED-NEXT: [[FACTOR_OP_MUL:%.*]] = mul i64 [[D3]], [[DELTA]] +; NOT_CONSTRAINED-NEXT: [[FACTOR_OP_MUL1:%.*]] = mul i64 [[D1]], [[DELTA]] +; NOT_CONSTRAINED-NEXT: [[FACTOR_OP_MUL2:%.*]] = mul i64 [[D2]], [[DELTA]] +; NOT_CONSTRAINED-NEXT: br label [[FOR_COND:%.*]] +; NOT_CONSTRAINED: for.cond: +; NOT_CONSTRAINED-NEXT: [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_J_1:%.*]], [[FOR_BODY:%.*]] ] +; NOT_CONSTRAINED-NEXT: [[CMP_NOT:%.*]] = icmp sgt i32 [[J]], [[I]] +; NOT_CONSTRAINED-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; NOT_CONSTRAINED: for.body: +; NOT_CONSTRAINED-NEXT: [[ADD_J_1]] = add nuw nsw i32 [[J]], 1 +; NOT_CONSTRAINED-NEXT: [[IDXPROM_J_1:%.*]] = zext i32 [[ADD_J_1]] to i64 +; NOT_CONSTRAINED-NEXT: [[ARRAYIDX_J_1:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_1]] +; NOT_CONSTRAINED-NEXT: [[CELL_1:%.*]] = load i64, ptr [[ARRAYIDX_J_1]], align 8 +; NOT_CONSTRAINED-NEXT: [[MUL_1:%.*]] = mul i64 [[CELL_1]], [[FACTOR_OP_MUL1]] +; NOT_CONSTRAINED-NEXT: [[IDXPROM_J:%.*]] = zext i32 [[J]] to i64 +; NOT_CONSTRAINED-NEXT: [[ARRAYIDX_J:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J]] +; NOT_CONSTRAINED-NEXT: [[CELL_2:%.*]] = load i64, ptr [[ARRAYIDX_J]], align 8 +; NOT_CONSTRAINED-NEXT: [[MUL_2:%.*]] = mul i64 [[CELL_2]], [[FACTOR_OP_MUL2]] +; NOT_CONSTRAINED-NEXT: [[ADD_J_2:%.*]] = add nuw nsw i32 [[J]], 2 +; NOT_CONSTRAINED-NEXT: [[IDXPROM_J_2:%.*]] = zext i32 [[ADD_J_2]] to i64 +; NOT_CONSTRAINED-NEXT: [[ARRAYIDX_J_2:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_2]] +; NOT_CONSTRAINED-NEXT: [[CELL_3:%.*]] = load i64, ptr [[ARRAYIDX_J_2]], align 8 +; NOT_CONSTRAINED-NEXT: [[MUL_3:%.*]] = mul i64 [[CELL_3]], [[FACTOR_OP_MUL]] +; NOT_CONSTRAINED-NEXT: [[REASS_ADD:%.*]] = add i64 [[MUL_2]], [[MUL_1]] +; NOT_CONSTRAINED-NEXT: [[REASS_ADD1:%.*]] = add i64 [[REASS_ADD]], [[MUL_3]] +; NOT_CONSTRAINED-NEXT: store i64 [[REASS_ADD1]], ptr [[ARRAYIDX_J_2]], align 8 +; NOT_CONSTRAINED-NEXT: br label [[FOR_COND]] +; NOT_CONSTRAINED: for.end: +; NOT_CONSTRAINED-NEXT: ret void +; +; CONSTRAINED-LABEL: define void @innermost_loop_3d +; CONSTRAINED-SAME: (i32 [[I:%.*]], i64 [[D1:%.*]], i64 [[D2:%.*]], i64 [[D3:%.*]], i64 [[DELTA:%.*]], ptr [[CELLS:%.*]]) { +; CONSTRAINED-NEXT: entry: +; CONSTRAINED-NEXT: br label [[FOR_COND:%.*]] +; CONSTRAINED: for.cond: +; CONSTRAINED-NEXT: [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_J_1:%.*]], [[FOR_BODY:%.*]] ] +; CONSTRAINED-NEXT: [[CMP_NOT:%.*]] = icmp sgt i32 [[J]], [[I]] +; CONSTRAINED-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; CONSTRAINED: for.body: +; CONSTRAINED-NEXT: [[ADD_J_1]] = add nuw nsw i32 [[J]], 1 +; CONSTRAINED-NEXT: [[IDXPROM_J_1:%.*]] = zext i32 [[ADD_J_1]] to i64 +; CONSTRAINED-NEXT: [[ARRAYIDX_J_1:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_1]] +; CONSTRAINED-NEXT: [[CELL_1:%.*]] = load i64, ptr [[ARRAYIDX_J_1]], align 8 +; CONSTRAINED-NEXT: [[MUL_1:%.*]] = mul i64 [[CELL_1]], [[D1]] +; CONSTRAINED-NEXT: [[IDXPROM_J:%.*]] = zext i32 [[J]] to i64 +; CONSTRAINED-NEXT: [[ARRAYIDX_J:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J]] +; CONSTRAINED-NEXT: [[CELL_2:%.*]] = load i64, ptr [[ARRAYIDX_J]], align 8 +; CONSTRAINED-NEXT: [[MUL_2:%.*]] = mul i64 [[CELL_2]], [[D2]] +; CONSTRAINED-NEXT: [[ADD_J_2:%.*]] = add nuw nsw i32 [[J]], 2 +; CONSTRAINED-NEXT: [[IDXPROM_J_2:%.*]] = zext i32 [[ADD_J_2]] to i64 +; CONSTRAINED-NEXT: [[ARRAYIDX_J_2:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_2]] +; CONSTRAINED-NEXT: [[CELL_3:%.*]] = load i64, ptr [[ARRAYIDX_J_2]], align 8 +; CONSTRAINED-NEXT: [[MUL_3:%.*]] = mul i64 [[CELL_3]], [[D3]] +; CONSTRAINED-NEXT: [[REASS_ADD:%.*]] = add i64 [[MUL_2]], [[MUL_1]] +; CONSTRAINED-NEXT: [[REASS_ADD1:%.*]] = add i64 [[REASS_ADD]], [[MUL_3]] +; CONSTRAINED-NEXT: [[REASS_MUL:%.*]] = mul i64 [[REASS_ADD1]], [[DELTA]] +; CONSTRAINED-NEXT: store i64 [[REASS_MUL]], ptr [[ARRAYIDX_J_2]], align 8 +; CONSTRAINED-NEXT: br label [[FOR_COND]] +; CONSTRAINED: for.end: +; CONSTRAINED-NEXT: ret void +; +entry: + br label %for.cond + +for.cond: + %j = phi i32 [ 0, %entry ], [ %add.j.1, %for.body ] + %cmp.not = icmp sgt i32 %j, %i + br i1 %cmp.not, label %for.end, label %for.body + +for.body: + %add.j.1 = add nuw nsw i32 %j, 1 + %idxprom.j.1 = zext i32 %add.j.1 to i64 + %arrayidx.j.1 = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j.1 + %cell.1 = load i64, ptr %arrayidx.j.1, align 8 + %mul.1 = mul i64 %cell.1, %d1 + %idxprom.j = zext i32 %j to i64 + %arrayidx.j = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j + %cell.2 = load i64, ptr %arrayidx.j, align 8 + %mul.2 = mul i64 %cell.2, %d2 + %add.j.2 = add nuw nsw i32 %j, 2 + %idxprom.j.2 = zext i32 %add.j.2 to i64 + %arrayidx.j.2 = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j.2 + %cell.3 = load i64, ptr %arrayidx.j.2, align 8 + %mul.3 = mul i64 %cell.3, %d3 + %reass.add = add i64 %mul.2, %mul.1 + %reass.add1 = add i64 %reass.add, %mul.3 + %reass.mul = mul i64 %reass.add1, %delta + store i64 %reass.mul, ptr %arrayidx.j.2, align 8 + br label %for.cond + +for.end: + ret void +} + +; +; The following loop will not be modified by the LICM pass: +; +; int j; +; +; for (j = 0; j <= i; j++) +; cells[j] = (d1 * cells[j + 1] + d2 * cells[j] + +; cells[j] * cells[j + 1]) * delta; +; +; This case differs as one of the multiplications involves no invariants. +; + +define void @innermost_loop_3d_reassociated_different(i32 %i, i64 %d1, i64 %d2, i64 %delta, ptr %cells) { +; CHECK-LABEL: define void @innermost_loop_3d_reassociated_different +; CHECK-SAME: (i32 [[I:%.*]], i64 [[D1:%.*]], i64 [[D2:%.*]], i64 [[DELTA:%.*]], ptr [[CELLS:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[FOR_COND:%.*]] +; CHECK: for.cond: +; CHECK-NEXT: [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_J_1:%.*]], [[FOR_BODY:%.*]] ] +; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp sgt i32 [[J]], [[I]] +; CHECK-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; CHECK: for.body: +; CHECK-NEXT: [[ADD_J_1]] = add nuw nsw i32 [[J]], 1 +; CHECK-NEXT: [[IDXPROM_J_1:%.*]] = zext i32 [[ADD_J_1]] to i64 +; CHECK-NEXT: [[ARRAYIDX_J_1:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_1]] +; CHECK-NEXT: [[CELL_1:%.*]] = load i64, ptr [[ARRAYIDX_J_1]], align 8 +; CHECK-NEXT: [[IDXPROM_J_2:%.*]] = zext i32 [[ADD_J_1]] to i64 +; CHECK-NEXT: [[ARRAYIDX_J_2:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J_2]] +; CHECK-NEXT: [[CELL_2:%.*]] = load i64, ptr [[ARRAYIDX_J_2]], align 8 +; CHECK-NEXT: [[CELL_3:%.*]] = load i64, ptr [[ARRAYIDX_J_2]], align 8 +; CHECK-NEXT: [[IDXPROM_J:%.*]] = zext i32 [[J]] to i64 +; CHECK-NEXT: [[ARRAYIDX_J:%.*]] = getelementptr inbounds i64, ptr [[CELLS]], i64 [[IDXPROM_J]] +; CHECK-NEXT: [[CELL_4:%.*]] = load i64, ptr [[ARRAYIDX_J]], align 8 +; CHECK-NEXT: [[MUL_1:%.*]] = mul i64 [[CELL_1]], [[D1]] +; CHECK-NEXT: [[MUL_2:%.*]] = mul i64 [[CELL_4]], [[D2]] +; CHECK-NEXT: [[EXTRA_MUL:%.*]] = mul i64 [[CELL_3]], [[CELL_2]] +; CHECK-NEXT: [[REASS_ADD:%.*]] = add i64 [[EXTRA_MUL]], [[MUL_1]] +; CHECK-NEXT: [[EXTRA_ADD:%.*]] = add i64 [[REASS_ADD]], [[MUL_2]] +; CHECK-NEXT: [[REASS_MUL:%.*]] = mul i64 [[EXTRA_ADD]], [[DELTA]] +; CHECK-NEXT: store i64 [[REASS_MUL]], ptr [[ARRAYIDX_J]], align 8 +; CHECK-NEXT: br label [[FOR_COND]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + br label %for.cond + +for.cond: + %j = phi i32 [ 0, %entry ], [ %add.j.1, %for.body ] + %cmp.not = icmp sgt i32 %j, %i + br i1 %cmp.not, label %for.end, label %for.body + +for.body: + %add.j.1 = add nuw nsw i32 %j, 1 + %idxprom.j.1 = zext i32 %add.j.1 to i64 + %arrayidx.j.1 = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j.1 + %cell.1 = load i64, ptr %arrayidx.j.1, align 8 + %idxprom.j.2 = zext i32 %add.j.1 to i64 + %arrayidx.j.2 = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j.2 + %cell.2 = load i64, ptr %arrayidx.j.2, align 8 + %idxprom.j.3 = zext i32 %add.j.1 to i64 + %cell.3 = load i64, ptr %arrayidx.j.2, align 8 + %idxprom.j = zext i32 %j to i64 + %arrayidx.j = getelementptr inbounds i64, ptr %cells, i64 %idxprom.j + %cell.4 = load i64, ptr %arrayidx.j, align 8 + %mul.1 = mul i64 %cell.1, %d1 + %mul.2 = mul i64 %cell.4, %d2 + %extra.mul = mul i64 %cell.3, %cell.2 + %reass.add = add i64 %extra.mul, %mul.1 + %extra.add = add i64 %reass.add, %mul.2 + %reass.mul = mul i64 %extra.add, %delta + store i64 %reass.mul, ptr %arrayidx.j, align 8 + br label %for.cond + +for.end: + ret void +} -- GitLab From bc8910d272e3f9ed58b21af2e7c7a9125e4b25ea Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 13 Feb 2024 13:01:18 +0800 Subject: [PATCH 010/284] [RISCV] Mark RISCVFoldMasks methods as const. NFC --- llvm/lib/Target/RISCV/RISCVFoldMasks.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp index 271d28f86a67..fddbaa97d063 100644 --- a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp +++ b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp @@ -47,10 +47,10 @@ public: StringRef getPassName() const override { return "RISC-V Fold Masks"; } private: - bool convertToUnmasked(MachineInstr &MI, MachineInstr *MaskDef); - bool convertVMergeToVMv(MachineInstr &MI, MachineInstr *MaskDef); + bool convertToUnmasked(MachineInstr &MI, MachineInstr *MaskDef) const; + bool convertVMergeToVMv(MachineInstr &MI, MachineInstr *MaskDef) const; - bool isAllOnesMask(MachineInstr *MaskDef); + bool isAllOnesMask(MachineInstr *MaskDef) const; }; } // namespace @@ -59,7 +59,7 @@ char RISCVFoldMasks::ID = 0; INITIALIZE_PASS(RISCVFoldMasks, DEBUG_TYPE, "RISC-V Fold Masks", false, false) -bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) { +bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) const { if (!MaskDef) return false; assert(MaskDef->isCopy() && MaskDef->getOperand(0).getReg() == RISCV::V0); @@ -89,7 +89,8 @@ bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) { // Transform (VMERGE_VVM_ false, false, true, allones, vl, sew) to // (VMV_V_V_ false, true, vl, sew). It may decrease uses of VMSET. -bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, MachineInstr *V0Def) { +bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, + MachineInstr *V0Def) const { #define CASE_VMERGE_TO_VMV(lmul) \ case RISCV::PseudoVMERGE_VVM_##lmul: \ NewOpc = RISCV::PseudoVMV_V_V_##lmul; \ @@ -133,7 +134,7 @@ bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, MachineInstr *V0Def) { } bool RISCVFoldMasks::convertToUnmasked(MachineInstr &MI, - MachineInstr *MaskDef) { + MachineInstr *MaskDef) const { const RISCV::RISCVMaskedPseudoInfo *I = RISCV::getMaskedPseudoInfo(MI.getOpcode()); if (!I) -- GitLab From b56b3d75b04cda762ac8e15b7ec6fa4f52a6735a Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Tue, 13 Feb 2024 13:34:27 +0800 Subject: [PATCH 011/284] [Clang][Sema] Don't consider top-level cv-qualifiers in template partial orderings (#81449) This fixes a regression since https://github.com/llvm/llvm-project/commit/340eac01f7dad6c24cee35dd35f2484098dd6b1a, from which we compared function parameter types with cv-qualifiers taken into account. However, as per [dcl.fct]/p5: > After producing the list of parameter types, any top-level cv-qualifiers modifying > a parameter type are deleted when forming the function type. Thus, I think we should use `hasSameUnqualifiedType` for type comparison. This fixes https://github.com/llvm/llvm-project/issues/75404. --- clang/docs/ReleaseNotes.rst | 2 ++ clang/lib/Sema/SemaTemplateDeduction.cpp | 7 +++++-- .../test/CXX/over/over.match/over.match.best/p1-2a.cpp | 10 ++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index aa167d75c3bf..dd790236e03b 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -228,6 +228,8 @@ Bug Fixes to C++ Support or non-constant more accurately. Previously, only a subset of the initializer elements were considered, misclassifying some initializers as constant. Fixes some of (`#80510 `). +- Clang now ignores top-level cv-qualifiers on function parameters in template partial orderings. + (`#75404 `_) Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index 994c997b9f6a..47cc22310c4e 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -5642,9 +5642,12 @@ FunctionTemplateDecl *Sema::getMoreSpecializedTemplate( Sema::TPL_TemplateParamsEquivalent)) return nullptr; + // [dcl.fct]p5: + // Any top-level cv-qualifiers modifying a parameter type are deleted when + // forming the function type. for (unsigned i = 0; i < NumParams1; ++i) - if (!Context.hasSameType(FD1->getParamDecl(i)->getType(), - FD2->getParamDecl(i)->getType())) + if (!Context.hasSameUnqualifiedType(FD1->getParamDecl(i)->getType(), + FD2->getParamDecl(i)->getType())) return nullptr; // C++20 [temp.func.order]p6.3: diff --git a/clang/test/CXX/over/over.match/over.match.best/p1-2a.cpp b/clang/test/CXX/over/over.match/over.match.best/p1-2a.cpp index dae1ba760cc2..db3e3e3bc859 100644 --- a/clang/test/CXX/over/over.match/over.match.best/p1-2a.cpp +++ b/clang/test/CXX/over/over.match/over.match.best/p1-2a.cpp @@ -97,13 +97,16 @@ namespace non_template static_assert(is_same_v()), int>); // expected-error {{call to 'baz' is ambiguous}} static_assert(is_same_v()), void>); // expected-error {{call to 'bar' is ambiguous}} + // Top-level cv-qualifiers are ignored in template partial ordering per [dcl.fct]/p5. + // After producing the list of parameter types, any top-level cv-qualifiers modifying + // a parameter type are deleted when forming the function type. template - constexpr int goo(int a) requires AtLeast2 && true { // expected-note {{candidate function}} + constexpr int goo(T a) requires AtLeast2 && true { return 1; } template - constexpr int goo(const int b) requires AtLeast2 { // expected-note {{candidate function}} + constexpr int goo(const T b) requires AtLeast2 { return 2; } @@ -122,7 +125,6 @@ namespace non_template return 2; } - // By temp.func.order-6.2.2, this is ambiguous because parameter a and b have different types. - static_assert(goo(1) == 1); // expected-error {{call to 'goo' is ambiguous}} + static_assert(goo(1) == 1); static_assert(doo(2) == 1); } -- GitLab From 85e6e71eb09cb9e75bbd475e5f54f876653c3f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Tue, 13 Feb 2024 06:56:58 +0100 Subject: [PATCH 012/284] [clang][Interp] Handle discarded PointerToIntegral casts Resolve an old TODO comment. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 4 +++- clang/test/AST/Interp/c.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 8a2c1e54e10a..ba6c1d5f5c94 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -167,7 +167,9 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) { return this->emitNull(classifyPrim(CE->getType()), CE); case CK_PointerToIntegral: { - // TODO: Discard handling. + if (DiscardResult) + return this->discard(SubExpr); + if (!this->visit(SubExpr)) return false; diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c index 392b682afd60..85c195d33a96 100644 --- a/clang/test/AST/Interp/c.c +++ b/clang/test/AST/Interp/c.c @@ -24,6 +24,7 @@ _Static_assert(!!1, ""); int a = (1 == 1 ? 5 : 3); _Static_assert(a == 5, ""); // all-error {{not an integral constant expression}} +const int DiscardedPtrToIntCast = ((intptr_t)((void*)0), 0); // all-warning {{left operand of comma operator has no effect}} const int b = 3; _Static_assert(b == 3, ""); // pedantic-ref-warning {{not an integer constant expression}} \ -- GitLab From bb77047a3b93e332be9cb3d85ad658e0ffa25525 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 13 Feb 2024 14:29:08 +0800 Subject: [PATCH 013/284] [RISCV] Handle fixed length vectors with exact VLEN in loweringEXTRACT_SUBVECTOR (#79949) This is a revival of #65392. When we lower an extract_subvector, we extract the subregister that the subvector is contained in first and then do a vslidedown with LMUL=1. We can currently only do this for scalable vectors though because the index is scaled by vscale and thus we will know what subregister the subvector lies in. For fixed length vectors, the index isn't scaled by vscale and so the subvector could lie in any arbitrary subregister, so we have to do a vslidedown with the full LMUL. The exception to this is when we know the exact VLEN: in which case, we can still work out the exact subregister and do the LMUL=1 vslidedown on it. This patch handles this case by scaling the index by 1/vscale before computing the subregister, and extending the LMUL=1 path to handle fixed length vectors. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 66 ++++-- .../rvv/fixed-vectors-extract-subvector.ll | 205 +++++++++++------- 2 files changed, 181 insertions(+), 90 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 12c0cd53514d..5bbd8273c113 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -9727,12 +9727,15 @@ SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op, if (OrigIdx == 0) return Op; - // If the subvector vector is a fixed-length type, we cannot use subregister - // manipulation to simplify the codegen; we don't know which register of a - // LMUL group contains the specific subvector as we only know the minimum - // register size. Therefore we must slide the vector group down the full - // amount. - if (SubVecVT.isFixedLengthVector()) { + const unsigned MinVLen = Subtarget.getRealMinVLen(); + const unsigned MaxVLen = Subtarget.getRealMaxVLen(); + + // If the subvector vector is a fixed-length type and we don't know VLEN + // exactly, we cannot use subregister manipulation to simplify the codegen; we + // don't know which register of a LMUL group contains the specific subvector + // as we only know the minimum register size. Therefore we must slide the + // vector group down the full amount. + if (SubVecVT.isFixedLengthVector() && MinVLen != MaxVLen) { MVT ContainerVT = VecVT; if (VecVT.isFixedLengthVector()) { ContainerVT = getContainerForFixedLengthVector(VecVT); @@ -9764,19 +9767,47 @@ SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op, return DAG.getBitcast(Op.getValueType(), Slidedown); } + if (VecVT.isFixedLengthVector()) { + VecVT = getContainerForFixedLengthVector(VecVT); + Vec = convertToScalableVector(VecVT, Vec, DAG, Subtarget); + } + + MVT ContainerSubVecVT = SubVecVT; + if (SubVecVT.isFixedLengthVector()) + ContainerSubVecVT = getContainerForFixedLengthVector(SubVecVT); + unsigned SubRegIdx, RemIdx; - std::tie(SubRegIdx, RemIdx) = - RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( - VecVT, SubVecVT, OrigIdx, TRI); + // extract_subvector scales the index by vscale is the subvector is scalable, + // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if + // we have a fixed length subvector, we need to adjust the index by 1/vscale. + if (SubVecVT.isFixedLengthVector()) { + assert(MinVLen == MaxVLen); + unsigned Vscale = MinVLen / RISCV::RVVBitsPerBlock; + std::tie(SubRegIdx, RemIdx) = + RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( + VecVT, ContainerSubVecVT, OrigIdx / Vscale, TRI); + RemIdx = (RemIdx * Vscale) + (OrigIdx % Vscale); + } else { + std::tie(SubRegIdx, RemIdx) = + RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( + VecVT, ContainerSubVecVT, OrigIdx, TRI); + } // If the Idx has been completely eliminated then this is a subvector extract // which naturally aligns to a vector register. These can easily be handled // using subregister manipulation. - if (RemIdx == 0) + if (RemIdx == 0) { + if (SubVecVT.isFixedLengthVector()) { + Vec = DAG.getTargetExtractSubreg(SubRegIdx, DL, ContainerSubVecVT, Vec); + return convertFromScalableVector(SubVecVT, Vec, DAG, Subtarget); + } return Op; + } - // Else SubVecVT is a fractional LMUL and may need to be slid down. - assert(RISCVVType::decodeVLMUL(getLMUL(SubVecVT)).second); + // Else SubVecVT is a fractional LMUL and may need to be slid down: if + // SubVecVT was > M1 then the index would need to be a multiple of VLMAX, and + // so would divide exactly. + assert(RISCVVType::decodeVLMUL(getLMUL(ContainerSubVecVT)).second); // If the vector type is an LMUL-group type, extract a subvector equal to the // nearest full vector register type. @@ -9791,10 +9822,17 @@ SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op, // Slide this vector register down by the desired number of elements in order // to place the desired subvector starting at element 0. - SDValue SlidedownAmt = - DAG.getVScale(DL, XLenVT, APInt(XLenVT.getSizeInBits(), RemIdx)); + SDValue SlidedownAmt; + if (SubVecVT.isFixedLengthVector()) + SlidedownAmt = DAG.getConstant(RemIdx, DL, Subtarget.getXLenVT()); + else + SlidedownAmt = + DAG.getVScale(DL, XLenVT, APInt(XLenVT.getSizeInBits(), RemIdx)); auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget); + if (SubVecVT.isFixedLengthVector()) + VL = getVLOp(SubVecVT.getVectorNumElements(), InterSubVT, DL, DAG, + Subtarget); SDValue Slidedown = getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL); diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll index 8b51a3883042..7d29b194f041 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll @@ -76,10 +76,8 @@ define void @extract_v1i32_v8i32_4(ptr %x, ptr %y) { ; CHECK-KNOWNVLEN128-LABEL: extract_v1i32_v8i32_4: ; CHECK-KNOWNVLEN128: # %bb.0: ; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0) -; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 4 ; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1) +; CHECK-KNOWNVLEN128-NEXT: vse32.v v9, (a1) ; CHECK-KNOWNVLEN128-NEXT: ret %a = load <8 x i32>, ptr %x %c = call <1 x i32> @llvm.vector.extract.v1i32.v8i32(<8 x i32> %a, i64 4) @@ -101,8 +99,8 @@ define void @extract_v1i32_v8i32_5(ptr %x, ptr %y) { ; CHECK-KNOWNVLEN128-LABEL: extract_v1i32_v8i32_5: ; CHECK-KNOWNVLEN128: # %bb.0: ; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0) -; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, m2, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 5 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v9, 1 ; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1) ; CHECK-KNOWNVLEN128-NEXT: ret @@ -172,10 +170,8 @@ define void @extract_v2i32_v8i32_4(ptr %x, ptr %y) { ; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_v8i32_4: ; CHECK-KNOWNVLEN128: # %bb.0: ; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0) -; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m2, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 4 ; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1) +; CHECK-KNOWNVLEN128-NEXT: vse32.v v9, (a1) ; CHECK-KNOWNVLEN128-NEXT: ret %a = load <8 x i32>, ptr %x %c = call <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32> %a, i64 4) @@ -197,8 +193,8 @@ define void @extract_v2i32_v8i32_6(ptr %x, ptr %y) { ; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_v8i32_6: ; CHECK-KNOWNVLEN128: # %bb.0: ; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0) -; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m2, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 6 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m1, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v9, 2 ; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1) ; CHECK-KNOWNVLEN128-NEXT: ret @@ -234,39 +230,59 @@ define void @extract_v2i32_nxv16i32_2( %x, ptr %y) { } define void @extract_v2i32_nxv16i32_4( %x, ptr %y) { -; CHECK-LABEL: extract_v2i32_nxv16i32_4: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 4 -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: ret +; CHECK-V-LABEL: extract_v2i32_nxv16i32_4: +; CHECK-V: # %bb.0: +; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, ta, ma +; CHECK-V-NEXT: vslidedown.vi v8, v8, 4 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vse32.v v8, (a0) +; CHECK-V-NEXT: ret +; +; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_nxv16i32_4: +; CHECK-KNOWNVLEN128: # %bb.0: +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vse32.v v9, (a0) +; CHECK-KNOWNVLEN128-NEXT: ret %c = call <2 x i32> @llvm.vector.extract.v2i32.nxv16i32( %x, i64 4) store <2 x i32> %c, ptr %y ret void } define void @extract_v2i32_nxv16i32_6( %x, ptr %y) { -; CHECK-LABEL: extract_v2i32_nxv16i32_6: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 6 -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: ret +; CHECK-V-LABEL: extract_v2i32_nxv16i32_6: +; CHECK-V: # %bb.0: +; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, ta, ma +; CHECK-V-NEXT: vslidedown.vi v8, v8, 6 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vse32.v v8, (a0) +; CHECK-V-NEXT: ret +; +; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_nxv16i32_6: +; CHECK-KNOWNVLEN128: # %bb.0: +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m1, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v9, 2 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a0) +; CHECK-KNOWNVLEN128-NEXT: ret %c = call <2 x i32> @llvm.vector.extract.v2i32.nxv16i32( %x, i64 6) store <2 x i32> %c, ptr %y ret void } define void @extract_v2i32_nxv16i32_8( %x, ptr %y) { -; CHECK-LABEL: extract_v2i32_nxv16i32_8: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 2, e32, m4, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 8 -; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma -; CHECK-NEXT: vse32.v v8, (a0) -; CHECK-NEXT: ret +; CHECK-V-LABEL: extract_v2i32_nxv16i32_8: +; CHECK-V: # %bb.0: +; CHECK-V-NEXT: vsetivli zero, 2, e32, m4, ta, ma +; CHECK-V-NEXT: vslidedown.vi v8, v8, 8 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vse32.v v8, (a0) +; CHECK-V-NEXT: ret +; +; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_nxv16i32_8: +; CHECK-KNOWNVLEN128: # %bb.0: +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vse32.v v10, (a0) +; CHECK-KNOWNVLEN128-NEXT: ret %c = call <2 x i32> @llvm.vector.extract.v2i32.nxv16i32( %x, i64 8) store <2 x i32> %c, ptr %y ret void @@ -333,9 +349,7 @@ define void @extract_v8i32_nxv16i32_8( %x, ptr %y) { ; ; CHECK-KNOWNVLEN128-LABEL: extract_v8i32_nxv16i32_8: ; CHECK-KNOWNVLEN128: # %bb.0: -; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 8, e32, m4, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 8 -; CHECK-KNOWNVLEN128-NEXT: vs2r.v v8, (a0) +; CHECK-KNOWNVLEN128-NEXT: vs2r.v v10, (a0) ; CHECK-KNOWNVLEN128-NEXT: ret %c = call <8 x i32> @llvm.vector.extract.v8i32.nxv16i32( %x, i64 8) store <8 x i32> %c, ptr %y @@ -611,9 +625,8 @@ define void @extract_v2i1_v64i1_42(ptr %x, ptr %y) { ; CHECK-KNOWNVLEN128-NEXT: vlm.v v0, (a0) ; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v8, 0 ; CHECK-KNOWNVLEN128-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-KNOWNVLEN128-NEXT: li a0, 42 -; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, m4, ta, ma -; CHECK-KNOWNVLEN128-NEXT: vslidedown.vx v8, v8, a0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, m1, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v10, 10 ; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, mf8, ta, ma ; CHECK-KNOWNVLEN128-NEXT: vmsne.vi v0, v8, 0 ; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v8, 0 @@ -741,51 +754,91 @@ define void @extract_v2i1_nxv64i1_2( %x, ptr %y) { } define void @extract_v2i1_nxv64i1_42( %x, ptr %y) { -; CHECK-LABEL: extract_v2i1_nxv64i1_42: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, m8, ta, ma -; CHECK-NEXT: vmv.v.i v8, 0 -; CHECK-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-NEXT: li a1, 42 -; CHECK-NEXT: vsetivli zero, 2, e8, m4, ta, ma -; CHECK-NEXT: vslidedown.vx v8, v8, a1 -; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma -; CHECK-NEXT: vmsne.vi v0, v8, 0 -; CHECK-NEXT: vmv.v.i v8, 0 -; CHECK-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.i v9, 0 -; CHECK-NEXT: vsetivli zero, 2, e8, mf2, tu, ma -; CHECK-NEXT: vmv.v.v v9, v8 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vmsne.vi v8, v9, 0 -; CHECK-NEXT: vsm.v v8, (a0) -; CHECK-NEXT: ret +; CHECK-V-LABEL: extract_v2i1_nxv64i1_42: +; CHECK-V: # %bb.0: +; CHECK-V-NEXT: vsetvli a1, zero, e8, m8, ta, ma +; CHECK-V-NEXT: vmv.v.i v8, 0 +; CHECK-V-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-V-NEXT: li a1, 42 +; CHECK-V-NEXT: vsetivli zero, 2, e8, m4, ta, ma +; CHECK-V-NEXT: vslidedown.vx v8, v8, a1 +; CHECK-V-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-V-NEXT: vmsne.vi v0, v8, 0 +; CHECK-V-NEXT: vmv.v.i v8, 0 +; CHECK-V-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-V-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-V-NEXT: vmv.v.i v9, 0 +; CHECK-V-NEXT: vsetivli zero, 2, e8, mf2, tu, ma +; CHECK-V-NEXT: vmv.v.v v9, v8 +; CHECK-V-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-V-NEXT: vmsne.vi v8, v9, 0 +; CHECK-V-NEXT: vsm.v v8, (a0) +; CHECK-V-NEXT: ret +; +; CHECK-KNOWNVLEN128-LABEL: extract_v2i1_nxv64i1_42: +; CHECK-KNOWNVLEN128: # %bb.0: +; CHECK-KNOWNVLEN128-NEXT: vsetvli a1, zero, e8, m8, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v8, 0 +; CHECK-KNOWNVLEN128-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, m1, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v10, 10 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmsne.vi v0, v8, 0 +; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v8, 0 +; CHECK-KNOWNVLEN128-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v9, 0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, mf2, tu, ma +; CHECK-KNOWNVLEN128-NEXT: vmv.v.v v9, v8 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmsne.vi v8, v9, 0 +; CHECK-KNOWNVLEN128-NEXT: vsm.v v8, (a0) +; CHECK-KNOWNVLEN128-NEXT: ret %c = call <2 x i1> @llvm.vector.extract.v2i1.nxv64i1( %x, i64 42) store <2 x i1> %c, ptr %y ret void } define void @extract_v2i1_nxv32i1_26( %x, ptr %y) { -; CHECK-LABEL: extract_v2i1_nxv32i1_26: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8, m4, ta, ma -; CHECK-NEXT: vmv.v.i v8, 0 -; CHECK-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 2, e8, m2, ta, ma -; CHECK-NEXT: vslidedown.vi v8, v8, 26 -; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma -; CHECK-NEXT: vmsne.vi v0, v8, 0 -; CHECK-NEXT: vmv.v.i v8, 0 -; CHECK-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vmv.v.i v9, 0 -; CHECK-NEXT: vsetivli zero, 2, e8, mf2, tu, ma -; CHECK-NEXT: vmv.v.v v9, v8 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vmsne.vi v8, v9, 0 -; CHECK-NEXT: vsm.v v8, (a0) -; CHECK-NEXT: ret +; CHECK-V-LABEL: extract_v2i1_nxv32i1_26: +; CHECK-V: # %bb.0: +; CHECK-V-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; CHECK-V-NEXT: vmv.v.i v8, 0 +; CHECK-V-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-V-NEXT: vsetivli zero, 2, e8, m2, ta, ma +; CHECK-V-NEXT: vslidedown.vi v8, v8, 26 +; CHECK-V-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-V-NEXT: vmsne.vi v0, v8, 0 +; CHECK-V-NEXT: vmv.v.i v8, 0 +; CHECK-V-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-V-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-V-NEXT: vmv.v.i v9, 0 +; CHECK-V-NEXT: vsetivli zero, 2, e8, mf2, tu, ma +; CHECK-V-NEXT: vmv.v.v v9, v8 +; CHECK-V-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-V-NEXT: vmsne.vi v8, v9, 0 +; CHECK-V-NEXT: vsm.v v8, (a0) +; CHECK-V-NEXT: ret +; +; CHECK-KNOWNVLEN128-LABEL: extract_v2i1_nxv32i1_26: +; CHECK-KNOWNVLEN128: # %bb.0: +; CHECK-KNOWNVLEN128-NEXT: vsetvli a1, zero, e8, m4, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v8, 0 +; CHECK-KNOWNVLEN128-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, m1, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v9, 10 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmsne.vi v0, v8, 0 +; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v8, 0 +; CHECK-KNOWNVLEN128-NEXT: vmerge.vim v8, v8, 1, v0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmv.v.i v9, 0 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e8, mf2, tu, ma +; CHECK-KNOWNVLEN128-NEXT: vmv.v.v v9, v8 +; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-KNOWNVLEN128-NEXT: vmsne.vi v8, v9, 0 +; CHECK-KNOWNVLEN128-NEXT: vsm.v v8, (a0) +; CHECK-KNOWNVLEN128-NEXT: ret %c = call <2 x i1> @llvm.vector.extract.v2i1.nxv32i1( %x, i64 26) store <2 x i1> %c, ptr %y ret void -- GitLab From 4bcbeaed63af8f1e18c1c74d3ef2a5a6ff7a22ac Mon Sep 17 00:00:00 2001 From: Austin Kerbow Date: Mon, 12 Feb 2024 22:33:29 -0800 Subject: [PATCH 014/284] [AMDGPU] Enable kernel arg preloading with gfx90a (#81180) Add a trap instruction to the beginning of the kernel prologue to handle cases where preloading is attempted on HW loaded with incompatible firmware. --- llvm/docs/AMDGPUUsage.rst | 5 +- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 3 +- .../AMDGPU/AMDGPULowerKernelArguments.cpp | 1 - llvm/lib/Target/AMDGPU/GCNSubtarget.h | 6 - .../MCTargetDesc/AMDGPUTargetStreamer.cpp | 16 +- .../MCTargetDesc/AMDGPUTargetStreamer.h | 9 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 3 +- .../CodeGen/AMDGPU/preload-kernarg-header.ll | 9 +- llvm/test/CodeGen/AMDGPU/preload-kernargs.ll | 15978 +++++++++++----- 9 files changed, 10685 insertions(+), 5345 deletions(-) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index ddd5fd487669..2b4ca0a3c84e 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -5515,7 +5515,10 @@ additional 256 bytes to the kernel_code_entry_byte_offset. This addition facilitates the incorporation of a prologue to the kernel entry to handle cases where code designed for kernarg preloading is executed on hardware equipped with incompatible firmware. If hardware has compatible firmware the 256 bytes at the -start of the kernel entry will be skipped. +start of the kernel entry will be skipped. Additionally, the compiler backend +may insert a trap instruction at the start of the kernel prologue to manage +situations where kernarg preloading is attempted on hardware with incompatible +firmware. .. _amdgpu-amdhsa-kernel-prolog: diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 5777a7cabb39..37a36b26b947 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -204,7 +204,8 @@ void AMDGPUAsmPrinter::emitFunctionBodyStart() { if (MFI.getNumKernargPreloadedSGPRs() > 0) { assert(AMDGPU::hasKernargPreload(STM)); - getTargetStreamer()->EmitKernargPreloadHeader(*getGlobalSTI()); + getTargetStreamer()->EmitKernargPreloadHeader(*getGlobalSTI(), + STM.isAmdHsaOS()); } } diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp index 015c71080d67..bc58407a7329 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp @@ -145,7 +145,6 @@ static bool lowerKernelArguments(Function &F, const TargetMachine &TM) { // Try to preload this argument into user SGPRs. if (Arg.hasInRegAttr() && InPreloadSequence && ST.hasKernargPreload() && - !ST.needsKernargPreloadBackwardsCompatibility() && !Arg.getType()->isAggregateType()) if (PreloadInfo.tryAllocPreloadSGPRs(AllocSize, EltOffset, LastExplicitArgOffset)) diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index b13b4f7d8f9a..c8e1b153e473 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -1258,12 +1258,6 @@ public: // \returns true if preloading kernel arguments is supported. bool hasKernargPreload() const { return KernargPreload; } - // \returns true if we need to generate backwards compatible code when - // preloading kernel arguments. - bool needsKernargPreloadBackwardsCompatibility() const { - return hasKernargPreload() && !hasGFX940Insts(); - } - // \returns true if the target has split barriers feature bool hasSplitBarriers() const { return getGeneration() >= GFX12; } diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index a25622c06bfa..4742b0b3e52e 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -782,18 +782,26 @@ bool AMDGPUTargetELFStreamer::EmitHSAMetadata(msgpack::Document &HSAMetadataDoc, } bool AMDGPUTargetAsmStreamer::EmitKernargPreloadHeader( - const MCSubtargetInfo &STI) { - for (int i = 0; i < 64; ++i) { + const MCSubtargetInfo &STI, bool TrapEnabled) { + const char *TrapInstr = TrapEnabled ? "\ts_trap 2" : "\ts_endpgm"; + OS << TrapInstr + << " ; Trap with incompatible firmware that doesn't " + "support preloading kernel arguments.\n"; + for (int i = 0; i < 63; ++i) { OS << "\ts_nop 0\n"; } return true; } bool AMDGPUTargetELFStreamer::EmitKernargPreloadHeader( - const MCSubtargetInfo &STI) { + const MCSubtargetInfo &STI, bool TrapEnabled) { const uint32_t Encoded_s_nop = 0xbf800000; + const uint32_t Encoded_s_trap = 0xbf920002; + const uint32_t Encoded_s_endpgm = 0xbf810000; + const uint32_t TrapInstr = TrapEnabled ? Encoded_s_trap : Encoded_s_endpgm; MCStreamer &OS = getStreamer(); - for (int i = 0; i < 64; ++i) { + OS.emitInt32(TrapInstr); + for (int i = 0; i < 63; ++i) { OS.emitInt32(Encoded_s_nop); } return true; diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h index ad5f27a33fcb..5aa80ff578c6 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h @@ -89,7 +89,8 @@ public: virtual bool EmitCodeEnd(const MCSubtargetInfo &STI) { return true; } /// \returns True on success, false on failure. - virtual bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI) { + virtual bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, + bool TrapEnabled) { return true; } @@ -146,7 +147,8 @@ public: bool EmitCodeEnd(const MCSubtargetInfo &STI) override; /// \returns True on success, false on failure. - bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI) override; + bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, + bool TrapEnabled) override; void EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, @@ -200,7 +202,8 @@ public: bool EmitCodeEnd(const MCSubtargetInfo &STI) override; /// \returns True on success, false on failure. - bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI) override; + bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, + bool TrapEnabled) override; void EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index a64a9e608f21..83221f7ead37 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -2826,8 +2826,7 @@ SDValue SITargetLowering::LowerFormalArguments( if (IsEntryFunc) { allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info); allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info); - if (IsKernel && Subtarget->hasKernargPreload() && - !Subtarget->needsKernargPreloadBackwardsCompatibility()) + if (IsKernel && Subtarget->hasKernargPreload()) allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, *TRI, *Info); allocateLDSKernelId(CCInfo, MF, *TRI, *Info); diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll b/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll index 75feac35dacd..a70488a00db7 100644 --- a/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll +++ b/llvm/test/CodeGen/AMDGPU/preload-kernarg-header.ll @@ -1,8 +1,11 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,HSA %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,HSA %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -filetype=obj < %s | llvm-objdump --arch=amdgcn --mcpu=gfx940 --disassemble - | FileCheck -check-prefixes=GCN,NON-HSA %s ; GCN: preload_kernarg_header -; GCN-COUNT-64: s_nop 0 +; HSA: s_trap 2 +; NON-HSA: s_endpgm +; GCN-COUNT-63: s_nop 0 define amdgpu_kernel void @preload_kernarg_header(ptr %arg) { store ptr %arg, ptr %arg ret void diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll index 57980214e58e..d20c3a4007ff 100644 --- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll +++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll @@ -1,1856 +1,3681 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=NO-PRELOAD %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-1 %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-2 %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-4 %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-8 %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NO-PRELOAD %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-1 %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-2 %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-4 %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-8 %s + +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-NO-PRELOAD %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-1 %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-2 %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-4 %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-8 %s define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) { -; NO-PRELOAD-LABEL: ptr1_i8: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: s_and_b32 s0, s4, 0xff -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: ptr1_i8: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xff -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: ptr1_i8: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_and_b32 s0, s4, 0xff -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: ptr1_i8: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_and_b32 s0, s4, 0xff -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: ptr1_i8: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_and_b32 s0, s4, 0xff -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: ptr1_i8: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: s_and_b32 s0, s4, 0xff +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: ptr1_i8: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xff +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: ptr1_i8: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_and_b32 s0, s4, 0xff +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: ptr1_i8: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_and_b32 s0, s4, 0xff +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: ptr1_i8: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_and_b32 s0, s4, 0xff +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: ptr1_i8: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: s_and_b32 s2, s2, 0xff +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: ptr1_i8: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xff +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: ptr1_i8: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_and_b32 s0, s8, 0xff +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: ptr1_i8: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_and_b32 s0, s8, 0xff +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: ptr1_i8: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_and_b32 s0, s8, 0xff +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm %ext = zext i8 %arg0 to i32 store i32 %ext, ptr addrspace(1) %out ret void } define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %arg0) { -; NO-PRELOAD-LABEL: ptr1_i8_zext_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: s_and_b32 s0, s4, 0xff -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: ptr1_i8_zext_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xff -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: ptr1_i8_zext_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_mov_b32 s0, 0xffff -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: ptr1_i8_zext_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_mov_b32 s0, 0xffff -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: ptr1_i8_zext_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_mov_b32 s0, 0xffff -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: ptr1_i8_zext_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: s_and_b32 s0, s4, 0xff +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: ptr1_i8_zext_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xff +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: ptr1_i8_zext_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_mov_b32 s0, 0xffff +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: ptr1_i8_zext_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_mov_b32 s0, 0xffff +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: ptr1_i8_zext_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_mov_b32 s0, 0xffff +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: ptr1_i8_zext_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: s_and_b32 s2, s2, 0xff +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: ptr1_i8_zext_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xff +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: ptr1_i8_zext_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_mov_b32 s0, 0xffff +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: ptr1_i8_zext_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_mov_b32 s0, 0xffff +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: ptr1_i8_zext_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_mov_b32 s0, 0xffff +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm %ext = zext i8 %arg0 to i32 store i32 %ext, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0) { -; NO-PRELOAD-LABEL: ptr1_i16_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: s_and_b32 s0, s4, 0xffff -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: ptr1_i16_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xffff -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: ptr1_i16_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_and_b32 s0, s4, 0xffff -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: ptr1_i16_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_and_b32 s0, s4, 0xffff -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: ptr1_i16_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_and_b32 s0, s4, 0xffff -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: ptr1_i16_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: s_and_b32 s0, s4, 0xffff +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: ptr1_i16_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xffff +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: ptr1_i16_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_and_b32 s0, s4, 0xffff +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: ptr1_i16_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_and_b32 s0, s4, 0xffff +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: ptr1_i16_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_and_b32 s0, s4, 0xffff +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: ptr1_i16_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: s_and_b32 s2, s2, 0xffff +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: ptr1_i16_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xffff +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: ptr1_i16_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_and_b32 s0, s8, 0xffff +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: ptr1_i16_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_and_b32 s0, s8, 0xffff +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: ptr1_i16_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_and_b32 s0, s8, 0xffff +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm %ext = zext i16 %arg0 to i32 store i32 %ext, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0) { -; NO-PRELOAD-LABEL: ptr1_i32_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s4 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: ptr1_i32_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: ptr1_i32_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: ptr1_i32_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: ptr1_i32_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: ptr1_i32_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: ptr1_i32_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: ptr1_i32_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: ptr1_i32_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: ptr1_i32_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: ptr1_i32_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: ptr1_i32_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: ptr1_i32_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: ptr1_i32_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: ptr1_i32_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store i32 %arg0, ptr addrspace(1) %out ret void } -; Check alignment on the second preloaded arg. define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) %out, i32 %arg1) { -; NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x10 -; NO-PRELOAD-NEXT: s_load_dword s5, s[0:1], 0x0 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: s_add_i32 s0, s5, s4 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s3, s[0:1], 0x10 -; PRELOAD-1-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: s_add_i32 s0, s2, s3 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_load_dword s0, s[0:1], 0x10 -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-2-NEXT: s_add_i32 s0, s2, s0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_add_i32 s0, s2, s6 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_add_i32 s0, s2, s6 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x10 +; GFX940-NO-PRELOAD-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: s_add_i32 s0, s5, s4 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s3, s[0:1], 0x10 +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: s_add_i32 s0, s2, s3 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_load_dword s0, s[0:1], 0x10 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-2-NEXT: s_add_i32 s0, s2, s0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_add_i32 s0, s2, s6 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_add_i32 s0, s2, s6 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x10 +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s3, s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: s_add_i32 s2, s3, s2 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s2, s[4:5], 0x10 +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: s_add_i32 s2, s6, s2 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_load_dword s0, s[4:5], 0x10 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-2-NEXT: s_add_i32 s0, s6, s0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[8:9] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_add_i32 s0, s6, s10 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[8:9] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_add_i32 s0, s6, s10 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[8:9] +; GFX90a-PRELOAD-8-NEXT: s_endpgm %add = add i32 %arg0, %arg1 store i32 %add, ptr addrspace(1) %out ret void } define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0, i16 %arg1) { -; NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: s_lshr_b32 s0, s4, 16 -; NO-PRELOAD-NEXT: s_and_b32 s1, s4, 0xffff -; NO-PRELOAD-NEXT: s_add_i32 s0, s1, s0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: s_lshr_b32 s1, s0, 16 -; PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xffff -; PRELOAD-1-NEXT: s_add_i32 s0, s0, s1 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-2-NEXT: s_and_b32 s1, s4, 0xffff -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-2-NEXT: s_lshr_b32 s0, s0, 16 -; PRELOAD-2-NEXT: s_add_i32 s0, s1, s0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-4-NEXT: s_and_b32 s1, s4, 0xffff -; PRELOAD-4-NEXT: s_add_i32 s0, s1, s0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-8-NEXT: s_and_b32 s1, s4, 0xffff -; PRELOAD-8-NEXT: s_add_i32 s0, s1, s0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-NO-PRELOAD-NEXT: s_and_b32 s1, s4, 0xffff +; GFX940-NO-PRELOAD-NEXT: s_add_i32 s0, s1, s0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: s_lshr_b32 s1, s0, 16 +; GFX940-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xffff +; GFX940-PRELOAD-1-NEXT: s_add_i32 s0, s0, s1 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-2-NEXT: s_and_b32 s1, s4, 0xffff +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s0, 16 +; GFX940-PRELOAD-2-NEXT: s_add_i32 s0, s1, s0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-4-NEXT: s_and_b32 s1, s4, 0xffff +; GFX940-PRELOAD-4-NEXT: s_add_i32 s0, s1, s0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-8-NEXT: s_and_b32 s1, s4, 0xffff +; GFX940-PRELOAD-8-NEXT: s_add_i32 s0, s1, s0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: s_lshr_b32 s3, s2, 16 +; GFX90a-NO-PRELOAD-NEXT: s_and_b32 s2, s2, 0xffff +; GFX90a-NO-PRELOAD-NEXT: s_add_i32 s2, s2, s3 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: s_lshr_b32 s1, s0, 16 +; GFX90a-PRELOAD-1-NEXT: s_and_b32 s0, s0, 0xffff +; GFX90a-PRELOAD-1-NEXT: s_add_i32 s0, s0, s1 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-2-NEXT: s_and_b32 s1, s8, 0xffff +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s0, 16 +; GFX90a-PRELOAD-2-NEXT: s_add_i32 s0, s1, s0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-4-NEXT: s_and_b32 s1, s8, 0xffff +; GFX90a-PRELOAD-4-NEXT: s_add_i32 s0, s1, s0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-8-NEXT: s_and_b32 s1, s8, 0xffff +; GFX90a-PRELOAD-8-NEXT: s_add_i32 s0, s1, s0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm %ext = zext i16 %arg0 to i32 %ext1 = zext i16 %arg1 to i32 %add = add i32 %ext, %ext1 @@ -1859,3563 +3684,7068 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 % } define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> %in) { -; NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s4 -; NO-PRELOAD-NEXT: global_store_short v0, v1, s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: ptr1_v2i8_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: global_store_short v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: ptr1_v2i8_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-2-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, 0 -; PRELOAD-2-NEXT: global_store_short v1, v0, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: ptr1_v2i8_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-4-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, 0 -; PRELOAD-4-NEXT: global_store_short v1, v0, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: ptr1_v2i8_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, 0 -; PRELOAD-8-NEXT: global_store_short v1, v0, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dword s4, s[0:1], 0x8 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-NO-PRELOAD-NEXT: global_store_short v0, v1, s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dword s0, s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: global_store_short v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, 0 +; GFX940-PRELOAD-2-NEXT: global_store_short v1, v0, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, 0 +; GFX940-PRELOAD-4-NEXT: global_store_short v1, v0, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, 0 +; GFX940-PRELOAD-8-NEXT: global_store_short v1, v0, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dword s2, s[4:5], 0x8 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_short v0, v1, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dword s0, s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_short v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, 0 +; GFX90a-PRELOAD-2-NEXT: global_store_short v1, v0, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, 0 +; GFX90a-PRELOAD-4-NEXT: global_store_short v1, v0, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, 0 +; GFX90a-PRELOAD-8-NEXT: global_store_short v1, v0, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <2 x i8> %in, ptr addrspace(1) %out ret void } -; Don't try to preload byref args. define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 %after.offset) { -; NO-PRELOAD-LABEL: byref_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x100 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s3 -; NO-PRELOAD-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 -; NO-PRELOAD-NEXT: s_waitcnt vmcnt(0) -; NO-PRELOAD-NEXT: global_store_dword v0, v2, s[4:5] sc0 sc1 -; NO-PRELOAD-NEXT: s_waitcnt vmcnt(0) -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: byref_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s1 -; PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-1-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: byref_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s1 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-2-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: byref_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s1 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-4-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: byref_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s1 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-8-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_waitcnt vmcnt(0) -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: byref_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x100 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s2 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s3 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt vmcnt(0) +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v2, s[4:5] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt vmcnt(0) +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: byref_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s1 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: byref_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s1 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: byref_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s1 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: byref_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x100 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s1 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_waitcnt vmcnt(0) +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: byref_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s1 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v1, s[2:3] +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt vmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v2, s[2:3] +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt vmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: byref_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s1 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v2, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: byref_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s1 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v2, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: byref_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s1 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v2, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: byref_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x100 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s1 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v2, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_waitcnt vmcnt(0) +; GFX90a-PRELOAD-8-NEXT: s_endpgm %in = load i32, ptr addrspace(4) %in.byref store volatile i32 %in, ptr addrspace(1) %out, align 4 store volatile i32 %after.offset, ptr addrspace(1) %out, align 4 ret void } -; TODO: Should do partial preload in cases like these where only part of the arg -; can be preloaded. define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> %in) nounwind { -; NO-PRELOAD-LABEL: v8i32_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v4, 0 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s8 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s9 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s10 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s11 -; NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1 -; NO-PRELOAD-NEXT: s_nop 1 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s7 -; NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v8i32_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 -; PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-1-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-1-NEXT: s_nop 1 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-1-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v8i32_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 -; PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-2-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-2-NEXT: s_nop 1 -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-2-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v8i32_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 -; PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-4-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-4-NEXT: s_nop 1 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-4-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v8i32_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 -; PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-8-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-8-NEXT: s_nop 1 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-8-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v8i32_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_nop 1 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v8i32_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_nop 1 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v8i32_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_nop 1 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v8i32_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_nop 1 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v8i32_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x20 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_nop 1 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v8i32_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 +; GFX90a-NO-PRELOAD-NEXT: s_nop 0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v8i32_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v8i32_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v8i32_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v8i32_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x20 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <8 x i32> %in, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 x i16> %in) nounwind { -; NO-PRELOAD-LABEL: v3i16_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 -; NO-PRELOAD-NEXT: global_store_short v0, v1, s[0:1] offset:4 sc0 sc1 -; NO-PRELOAD-NEXT: global_store_dword v0, v2, s[0:1] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v3i16_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s0 -; PRELOAD-1-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 -; PRELOAD-1-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v3i16_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-2-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v3i16_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-4-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v3i16_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-8-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 -; PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v3i16_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 +; GFX940-NO-PRELOAD-NEXT: global_store_short v0, v1, s[0:1] offset:4 sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v2, s[0:1] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v3i16_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s0 +; GFX940-PRELOAD-1-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v3i16_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-2-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-2-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v3i16_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-4-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-4-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v3i16_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-8-NEXT: global_store_short v0, v1, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s4 +; GFX940-PRELOAD-8-NEXT: global_store_dword v0, v1, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v3i16_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_short v0, v1, s[0:1] offset:4 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v2, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v3i16_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_short v0, v1, s[6:7] offset:4 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v2, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v3i16_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-2-NEXT: global_store_short v0, v1, s[6:7] offset:4 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v3i16_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-4-NEXT: global_store_short v0, v1, s[6:7] offset:4 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v3i16_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-8-NEXT: global_store_short v0, v1, s[6:7] offset:4 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s8 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v0, v1, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <3 x i16> %in, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 x i32> %in) nounwind { -; NO-PRELOAD-LABEL: v3i32_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v3, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 -; NO-PRELOAD-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v3i32_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 -; PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-1-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v3i32_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, s6 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s7 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s8 -; PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-2-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v3i32_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, s6 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s7 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s8 -; PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-4-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v3i32_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, s6 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s7 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s8 -; PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-8-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v3i32_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v3i32_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v3i32_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s6 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s7 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v3i32_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s6 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s7 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v3i32_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s6 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s7 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v3i32_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s1 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v3i32_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s0 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s2 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v3i32_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s10 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s11 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s12 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v3i32_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s10 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s11 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s12 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v3i32_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s10 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s11 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s12 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <3 x i32> %in, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 x float> %in) nounwind { -; NO-PRELOAD-LABEL: v3f32_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v3, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 -; NO-PRELOAD-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v3f32_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 -; PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-1-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v3f32_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, s6 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s7 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s8 -; PRELOAD-2-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v3f32_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, s6 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s7 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s8 -; PRELOAD-4-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v3f32_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, s6 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s7 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s8 -; PRELOAD-8-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v3f32_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v3f32_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x10 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v3f32_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s6 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s7 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v3f32_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s6 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s7 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v3f32_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s6 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s7 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v3f32_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s1 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v3f32_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s0 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s2 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v3f32_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s10 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s11 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s12 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v3f32_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s10 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s11 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s12 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v3f32_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s10 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s11 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s12 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx3 v3, v[0:2], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <3 x float> %in, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 x i8> %in) nounwind { -; NO-PRELOAD-LABEL: v5i8_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 -; NO-PRELOAD-NEXT: global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1 -; NO-PRELOAD-NEXT: global_store_dword v0, v2, s[0:1] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v5i8_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s0 -; PRELOAD-1-NEXT: global_store_byte v0, v1, s[2:3] offset:4 sc0 sc1 -; PRELOAD-1-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v5i8_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 24 -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-2-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s5 -; PRELOAD-2-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, 0 -; PRELOAD-2-NEXT: global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1 -; PRELOAD-2-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v5i8_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 24 -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-4-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s5 -; PRELOAD-4-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, 0 -; PRELOAD-4-NEXT: global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1 -; PRELOAD-4-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v5i8_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 24 -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s5 -; PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, 0 -; PRELOAD-8-NEXT: global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1 -; PRELOAD-8-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v5i8_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 +; GFX940-NO-PRELOAD-NEXT: global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: global_store_dword v0, v2, s[0:1] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v5i8_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s0 +; GFX940-PRELOAD-1-NEXT: global_store_byte v0, v1, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-1-NEXT: global_store_dword v0, v2, s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v5i8_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s5 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, 0 +; GFX940-PRELOAD-2-NEXT: global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-2-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v5i8_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s5 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, 0 +; GFX940-PRELOAD-4-NEXT: global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-4-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v5i8_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s5 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, 0 +; GFX940-PRELOAD-8-NEXT: global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1 +; GFX940-PRELOAD-8-NEXT: global_store_dword v1, v0, s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v5i8_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s2 +; GFX90a-NO-PRELOAD-NEXT: global_store_byte v0, v1, s[0:1] offset:4 +; GFX90a-NO-PRELOAD-NEXT: global_store_dword v0, v2, s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v5i8_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s1 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s0 +; GFX90a-PRELOAD-1-NEXT: global_store_byte v0, v1, s[6:7] offset:4 +; GFX90a-PRELOAD-1-NEXT: global_store_dword v0, v2, s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v5i8_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 24 +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s9 +; GFX90a-PRELOAD-2-NEXT: global_store_byte v1, v2, s[6:7] offset:4 +; GFX90a-PRELOAD-2-NEXT: global_store_dword v1, v0, s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v5i8_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 24 +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s9 +; GFX90a-PRELOAD-4-NEXT: global_store_byte v1, v2, s[6:7] offset:4 +; GFX90a-PRELOAD-4-NEXT: global_store_dword v1, v0, s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v5i8_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 24 +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s9 +; GFX90a-PRELOAD-8-NEXT: global_store_byte v1, v2, s[6:7] offset:4 +; GFX90a-PRELOAD-8-NEXT: global_store_dword v1, v0, s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <5 x i8> %in, ptr addrspace(1) %out, align 4 ret void } define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x double> %in) nounwind { -; NO-PRELOAD-LABEL: v5f64_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x60 -; NO-PRELOAD-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 -; NO-PRELOAD-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v4, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b64_e32 v[2:3], s[2:3] -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s8 -; NO-PRELOAD-NEXT: global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s9 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s10 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s11 -; NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1 -; NO-PRELOAD-NEXT: s_nop 1 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s7 -; NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v5f64_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 -; PRELOAD-1-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 -; PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b64_e32 v[2:3], s[12:13] -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-1-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-1-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-1-NEXT: s_nop 1 -; PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-1-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v5f64_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 -; PRELOAD-2-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 -; PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-2-NEXT: v_mov_b64_e32 v[2:3], s[12:13] -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-2-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-2-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-2-NEXT: s_nop 1 -; PRELOAD-2-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-2-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v5f64_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 -; PRELOAD-4-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 -; PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-4-NEXT: v_mov_b64_e32 v[2:3], s[12:13] -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-4-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-4-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-4-NEXT: s_nop 1 -; PRELOAD-4-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-4-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v5f64_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 -; PRELOAD-8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 -; PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 -; PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-8-NEXT: v_mov_b64_e32 v[2:3], s[12:13] -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 -; PRELOAD-8-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s10 -; PRELOAD-8-NEXT: v_mov_b32_e32 v3, s11 -; PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 -; PRELOAD-8-NEXT: s_nop 1 -; PRELOAD-8-NEXT: v_mov_b32_e32 v0, s4 -; PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, s6 -; PRELOAD-8-NEXT: v_mov_b32_e32 v3, s7 -; PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v5f64_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x60 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b64_e32 v[2:3], s[2:3] +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_nop 1 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v5f64_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 +; GFX940-PRELOAD-1-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b64_e32 v[2:3], s[12:13] +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_nop 1 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v5f64_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 +; GFX940-PRELOAD-2-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-2-NEXT: v_mov_b64_e32 v[2:3], s[12:13] +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_nop 1 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v5f64_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 +; GFX940-PRELOAD-4-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-4-NEXT: v_mov_b64_e32 v[2:3], s[12:13] +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_nop 1 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v5f64_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x60 +; GFX940-PRELOAD-8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x40 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-8-NEXT: v_mov_b64_e32 v[2:3], s[12:13] +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s10 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s11 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_nop 1 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s5 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s6 +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s7 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v5f64_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 +; GFX90a-NO-PRELOAD-NEXT: s_nop 0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v5f64_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] offset:32 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v5f64_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 +; GFX90a-PRELOAD-2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-2-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-2-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] offset:32 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v5f64_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 +; GFX90a-PRELOAD-4-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-4-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-4-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] offset:32 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v5f64_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x60 +; GFX90a-PRELOAD-8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x40 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v4, 0 +; GFX90a-PRELOAD-8-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s12 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] offset:32 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s13 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s14 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s15 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] offset:16 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v0, s8 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v1, s9 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, s10 +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v3, s11 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <5 x double> %in, ptr addrspace(1) %out, align 8 ret void } define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) { -; NO-PRELOAD-LABEL: v8i8_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b64_e32 v[0:1], s[2:3] -; NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: v8i8_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b64_e32 v[0:1], s[0:1] -; PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: v8i8_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 8 -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 24 -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 16 -; PRELOAD-2-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-2-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 24 -; PRELOAD-2-NEXT: v_lshlrev_b16_e64 v2, 8, s0 -; PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-2-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: v8i8_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 8 -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 24 -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 16 -; PRELOAD-4-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-4-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 24 -; PRELOAD-4-NEXT: v_lshlrev_b16_e64 v2, 8, s0 -; PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-4-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: v8i8_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 8 -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 24 -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 -; PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 16 -; PRELOAD-8-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 -; PRELOAD-8-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 24 -; PRELOAD-8-NEXT: v_lshlrev_b16_e64 v2, 8, s0 -; PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 -; PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: v8i8_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b64_e32 v[0:1], s[2:3] +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: v8i8_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX940-PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: v8i8_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 8 +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 24 +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s5, 16 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX940-PRELOAD-2-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: v8i8_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 8 +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 24 +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s5, 16 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX940-PRELOAD-4-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: v8i8_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 8 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 24 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s5, 16 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 8 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 24 +; GFX940-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX940-PRELOAD-8-NEXT: s_lshr_b32 s0, s4, 16 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: v8i8_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1] +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: v8i8_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: v8i8_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s9, 8 +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s9, 24 +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s9, 16 +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 24 +; GFX90a-PRELOAD-2-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX90a-PRELOAD-2-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: v8i8_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s9, 8 +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s9, 24 +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s9, 16 +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 24 +; GFX90a-PRELOAD-4-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX90a-PRELOAD-4-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: v8i8_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s9, 8 +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s9, 24 +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v1, 8, s0 +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s9, 16 +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 8 +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v0, 8, s0 +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 24 +; GFX90a-PRELOAD-8-NEXT: v_lshlrev_b16_e64 v2, 8, s0 +; GFX90a-PRELOAD-8-NEXT: s_lshr_b32 s0, s8, 16 +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store <8 x i8> %in, ptr addrspace(1) %out ret void } define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) { -; NO-PRELOAD-LABEL: i64_kernel_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s2 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 -; NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: i64_kernel_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b64_e32 v[0:1], s[0:1] -; PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: i64_kernel_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-2-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: i64_kernel_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-4-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: i64_kernel_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-8-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: i64_kernel_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s2 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: i64_kernel_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX940-PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: i64_kernel_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: i64_kernel_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: i64_kernel_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: i64_kernel_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s2 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: i64_kernel_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: i64_kernel_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-2-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: i64_kernel_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-4-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: i64_kernel_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store i64 %a, ptr addrspace(1) %out, align 8 ret void } define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double %in) { -; NO-PRELOAD-LABEL: f64_kernel_preload_arg: -; NO-PRELOAD: ; %bb.0: -; NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 -; NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) -; NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s2 -; NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 -; NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 -; NO-PRELOAD-NEXT: s_endpgm -; -; PRELOAD-1-LABEL: f64_kernel_preload_arg: -; PRELOAD-1: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: s_nop 0 -; PRELOAD-1-NEXT: ; %bb.0: -; PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 -; PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) -; PRELOAD-1-NEXT: v_mov_b64_e32 v[0:1], s[0:1] -; PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-1-NEXT: s_endpgm -; -; PRELOAD-2-LABEL: f64_kernel_preload_arg: -; PRELOAD-2: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: s_nop 0 -; PRELOAD-2-NEXT: ; %bb.0: -; PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-2-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-2-NEXT: s_endpgm -; -; PRELOAD-4-LABEL: f64_kernel_preload_arg: -; PRELOAD-4: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: s_nop 0 -; PRELOAD-4-NEXT: ; %bb.0: -; PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-4-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-4-NEXT: s_endpgm -; -; PRELOAD-8-LABEL: f64_kernel_preload_arg: -; PRELOAD-8: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: s_nop 0 -; PRELOAD-8-NEXT: ; %bb.0: -; PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 -; PRELOAD-8-NEXT: v_mov_b64_e32 v[0:1], s[4:5] -; PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 -; PRELOAD-8-NEXT: s_endpgm +; GFX940-NO-PRELOAD-LABEL: f64_kernel_preload_arg: +; GFX940-NO-PRELOAD: ; %bb.0: +; GFX940-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s2 +; GFX940-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX940-NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1 +; GFX940-NO-PRELOAD-NEXT: s_endpgm +; +; GFX940-PRELOAD-1-LABEL: f64_kernel_preload_arg: +; GFX940-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: s_nop 0 +; GFX940-PRELOAD-1-NEXT: ; %bb.0: +; GFX940-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8 +; GFX940-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-PRELOAD-1-NEXT: v_mov_b64_e32 v[0:1], s[0:1] +; GFX940-PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-1-NEXT: s_endpgm +; +; GFX940-PRELOAD-2-LABEL: f64_kernel_preload_arg: +; GFX940-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: s_nop 0 +; GFX940-PRELOAD-2-NEXT: ; %bb.0: +; GFX940-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-2-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-2-NEXT: s_endpgm +; +; GFX940-PRELOAD-4-LABEL: f64_kernel_preload_arg: +; GFX940-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: s_nop 0 +; GFX940-PRELOAD-4-NEXT: ; %bb.0: +; GFX940-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-4-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-4-NEXT: s_endpgm +; +; GFX940-PRELOAD-8-LABEL: f64_kernel_preload_arg: +; GFX940-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: s_nop 0 +; GFX940-PRELOAD-8-NEXT: ; %bb.0: +; GFX940-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 +; GFX940-PRELOAD-8-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1 +; GFX940-PRELOAD-8-NEXT: s_endpgm +; +; GFX90a-NO-PRELOAD-LABEL: f64_kernel_preload_arg: +; GFX90a-NO-PRELOAD: ; %bb.0: +; GFX90a-NO-PRELOAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-NO-PRELOAD-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v0, s2 +; GFX90a-NO-PRELOAD-NEXT: v_mov_b32_e32 v1, s3 +; GFX90a-NO-PRELOAD-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX90a-NO-PRELOAD-NEXT: s_endpgm +; +; GFX90a-PRELOAD-1-LABEL: f64_kernel_preload_arg: +; GFX90a-PRELOAD-1: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: s_nop 0 +; GFX90a-PRELOAD-1-NEXT: ; %bb.0: +; GFX90a-PRELOAD-1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; GFX90a-PRELOAD-1-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-1-NEXT: s_waitcnt lgkmcnt(0) +; GFX90a-PRELOAD-1-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1] +; GFX90a-PRELOAD-1-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-1-NEXT: s_endpgm +; +; GFX90a-PRELOAD-2-LABEL: f64_kernel_preload_arg: +; GFX90a-PRELOAD-2: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: s_nop 0 +; GFX90a-PRELOAD-2-NEXT: ; %bb.0: +; GFX90a-PRELOAD-2-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-2-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] +; GFX90a-PRELOAD-2-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-2-NEXT: s_endpgm +; +; GFX90a-PRELOAD-4-LABEL: f64_kernel_preload_arg: +; GFX90a-PRELOAD-4: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: s_nop 0 +; GFX90a-PRELOAD-4-NEXT: ; %bb.0: +; GFX90a-PRELOAD-4-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-4-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] +; GFX90a-PRELOAD-4-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-4-NEXT: s_endpgm +; +; GFX90a-PRELOAD-8-LABEL: f64_kernel_preload_arg: +; GFX90a-PRELOAD-8: s_trap 2 ; Trap with incompatible firmware that doesn't support preloading kernel arguments. +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: s_nop 0 +; GFX90a-PRELOAD-8-NEXT: ; %bb.0: +; GFX90a-PRELOAD-8-NEXT: v_mov_b32_e32 v2, 0 +; GFX90a-PRELOAD-8-NEXT: v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1] +; GFX90a-PRELOAD-8-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX90a-PRELOAD-8-NEXT: s_endpgm store double %in, ptr addrspace(1) %out ret void } -- GitLab From 785eddd7a786b798427e336b79bc0f2495a49984 Mon Sep 17 00:00:00 2001 From: sstipanovic <146831748+sstipanovic@users.noreply.github.com> Date: Tue, 13 Feb 2024 08:26:10 +0100 Subject: [PATCH 015/284] [AMDGPU][GlobalIsel] Introduce isRegisterClassType to check for legal types, instead of checking bit width. (#68189) In D151116 it was suggested to have a set of classes to cover every possible case. This does it for bitcast first. closes #79578 --- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 135 +++++++++++------- .../AMDGPU/GlobalISel/bitcast_38_i16.ll | 85 +++++++++++ .../AMDGPU/GlobalISel/extractelement.ll | 126 ++++++++++++++++ .../GlobalISel/legalize-build-vector.mir | 123 ---------------- 4 files changed, 296 insertions(+), 173 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/bitcast_38_i16.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 97952de3e6a3..def08cc89b41 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -239,6 +239,7 @@ static bool isRegisterVectorType(LLT Ty) { EltSize == 128 || EltSize == 256; } +// TODO: replace all uses of isRegisterType with isRegisterClassType static bool isRegisterType(LLT Ty) { if (!isRegisterSize(Ty.getSizeInBits())) return false; @@ -258,6 +259,8 @@ static LegalityPredicate isRegisterType(unsigned TypeIdx) { } // RegisterType that doesn't have a corresponding RegClass. +// TODO: Once `isRegisterType` is replaced with `isRegisterClassType` this +// should be removed. static LegalityPredicate isIllegalRegisterType(unsigned TypeIdx) { return [=](const LegalityQuery &Query) { LLT Ty = Query.Types[TypeIdx]; @@ -276,6 +279,85 @@ static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) { }; } +static const LLT S1 = LLT::scalar(1); +static const LLT S8 = LLT::scalar(8); +static const LLT S16 = LLT::scalar(16); +static const LLT S32 = LLT::scalar(32); +static const LLT S64 = LLT::scalar(64); +static const LLT S96 = LLT::scalar(96); +static const LLT S128 = LLT::scalar(128); +static const LLT S160 = LLT::scalar(160); +static const LLT S224 = LLT::scalar(224); +static const LLT S256 = LLT::scalar(256); +static const LLT S512 = LLT::scalar(512); +static const LLT MaxScalar = LLT::scalar(MaxRegisterSize); + +static const LLT V2S8 = LLT::fixed_vector(2, 8); +static const LLT V2S16 = LLT::fixed_vector(2, 16); +static const LLT V4S16 = LLT::fixed_vector(4, 16); +static const LLT V6S16 = LLT::fixed_vector(6, 16); +static const LLT V8S16 = LLT::fixed_vector(8, 16); +static const LLT V10S16 = LLT::fixed_vector(10, 16); +static const LLT V12S16 = LLT::fixed_vector(12, 16); +static const LLT V16S16 = LLT::fixed_vector(16, 16); + +static const LLT V2S32 = LLT::fixed_vector(2, 32); +static const LLT V3S32 = LLT::fixed_vector(3, 32); +static const LLT V4S32 = LLT::fixed_vector(4, 32); +static const LLT V5S32 = LLT::fixed_vector(5, 32); +static const LLT V6S32 = LLT::fixed_vector(6, 32); +static const LLT V7S32 = LLT::fixed_vector(7, 32); +static const LLT V8S32 = LLT::fixed_vector(8, 32); +static const LLT V9S32 = LLT::fixed_vector(9, 32); +static const LLT V10S32 = LLT::fixed_vector(10, 32); +static const LLT V11S32 = LLT::fixed_vector(11, 32); +static const LLT V12S32 = LLT::fixed_vector(12, 32); +static const LLT V16S32 = LLT::fixed_vector(16, 32); +static const LLT V32S32 = LLT::fixed_vector(32, 32); + +static const LLT V2S64 = LLT::fixed_vector(2, 64); +static const LLT V3S64 = LLT::fixed_vector(3, 64); +static const LLT V4S64 = LLT::fixed_vector(4, 64); +static const LLT V5S64 = LLT::fixed_vector(5, 64); +static const LLT V6S64 = LLT::fixed_vector(6, 64); +static const LLT V7S64 = LLT::fixed_vector(7, 64); +static const LLT V8S64 = LLT::fixed_vector(8, 64); +static const LLT V16S64 = LLT::fixed_vector(16, 64); + +static const LLT V2S128 = LLT::fixed_vector(2, 128); +static const LLT V4S128 = LLT::fixed_vector(4, 128); + +static std::initializer_list AllScalarTypes = {S32, S64, S96, S128, + S160, S224, S256, S512}; + +static std::initializer_list AllS16Vectors{ + V2S16, V4S16, V6S16, V8S16, V10S16, V12S16, V16S16, V2S128, V4S128}; + +static std::initializer_list AllS32Vectors = { + V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32, + V9S32, V10S32, V11S32, V12S32, V16S32, V32S32}; + +static std::initializer_list AllS64Vectors = {V2S64, V3S64, V4S64, V5S64, + V6S64, V7S64, V8S64, V16S64}; + +// Checks whether a type is in the list of legal register types. +static bool isRegisterClassType(LLT Ty) { + if (Ty.isVector() && Ty.getElementType().isPointer()) + Ty = LLT::fixed_vector(Ty.getNumElements(), + LLT::scalar(Ty.getScalarSizeInBits())); + else if (Ty.isPointer()) + Ty = LLT::scalar(Ty.getScalarSizeInBits()); + + return is_contained(AllS32Vectors, Ty) || is_contained(AllS64Vectors, Ty) || + is_contained(AllScalarTypes, Ty) || is_contained(AllS16Vectors, Ty); +} + +static LegalityPredicate isRegisterClassType(unsigned TypeIdx) { + return [TypeIdx](const LegalityQuery &Query) { + return isRegisterClassType(Query.Types[TypeIdx]); + }; +} + // If we have a truncating store or an extending load with a data size larger // than 32-bits, we need to reduce to a 32-bit type. static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) { @@ -578,52 +660,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, return LLT::pointer(AS, TM.getPointerSizeInBits(AS)); }; - const LLT S1 = LLT::scalar(1); - const LLT S8 = LLT::scalar(8); - const LLT S16 = LLT::scalar(16); - const LLT S32 = LLT::scalar(32); - const LLT S64 = LLT::scalar(64); - const LLT S128 = LLT::scalar(128); - const LLT S256 = LLT::scalar(256); - const LLT S512 = LLT::scalar(512); - const LLT MaxScalar = LLT::scalar(MaxRegisterSize); - - const LLT V2S8 = LLT::fixed_vector(2, 8); - const LLT V2S16 = LLT::fixed_vector(2, 16); - const LLT V4S16 = LLT::fixed_vector(4, 16); - - const LLT V2S32 = LLT::fixed_vector(2, 32); - const LLT V3S32 = LLT::fixed_vector(3, 32); - const LLT V4S32 = LLT::fixed_vector(4, 32); - const LLT V5S32 = LLT::fixed_vector(5, 32); - const LLT V6S32 = LLT::fixed_vector(6, 32); - const LLT V7S32 = LLT::fixed_vector(7, 32); - const LLT V8S32 = LLT::fixed_vector(8, 32); - const LLT V9S32 = LLT::fixed_vector(9, 32); - const LLT V10S32 = LLT::fixed_vector(10, 32); - const LLT V11S32 = LLT::fixed_vector(11, 32); - const LLT V12S32 = LLT::fixed_vector(12, 32); - const LLT V13S32 = LLT::fixed_vector(13, 32); - const LLT V14S32 = LLT::fixed_vector(14, 32); - const LLT V15S32 = LLT::fixed_vector(15, 32); - const LLT V16S32 = LLT::fixed_vector(16, 32); - const LLT V32S32 = LLT::fixed_vector(32, 32); - - const LLT V2S64 = LLT::fixed_vector(2, 64); - const LLT V3S64 = LLT::fixed_vector(3, 64); - const LLT V4S64 = LLT::fixed_vector(4, 64); - const LLT V5S64 = LLT::fixed_vector(5, 64); - const LLT V6S64 = LLT::fixed_vector(6, 64); - const LLT V7S64 = LLT::fixed_vector(7, 64); - const LLT V8S64 = LLT::fixed_vector(8, 64); - const LLT V16S64 = LLT::fixed_vector(16, 64); - - std::initializer_list AllS32Vectors = - {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32, - V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32, V32S32}; - std::initializer_list AllS64Vectors = - {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64}; - const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS); const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS); const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT); @@ -836,10 +872,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .scalarize(0); getActionDefinitionsBuilder(G_BITCAST) - // Don't worry about the size constraint. - .legalIf(all(isRegisterType(0), isRegisterType(1))) - .lower(); - + // Don't worry about the size constraint. + .legalIf(all(isRegisterClassType(0), isRegisterClassType(1))) + .lower(); getActionDefinitionsBuilder(G_CONSTANT) .legalFor({S1, S32, S64, S16, GlobalPtr, diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/bitcast_38_i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/bitcast_38_i16.ll new file mode 100644 index 000000000000..5bea13af1649 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/bitcast_38_i16.ll @@ -0,0 +1,85 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GPRIDX %s +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL %s +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s +; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s +define void @main(<19 x i32> %arg) { +; GCN-LABEL: main: +; GCN: ; %bb.0: ; %bb +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GCN-NEXT: s_mov_b32 s4, 0 +; GCN-NEXT: s_mov_b32 s12, s4 +; GCN-NEXT: v_cmp_eq_u16_e32 vcc, 0, v0 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: s_mov_b32 s13, s4 +; GCN-NEXT: v_mov_b32_e32 v4, s12 +; GCN-NEXT: s_mov_b32 s5, s4 +; GCN-NEXT: s_mov_b32 s6, s4 +; GCN-NEXT: s_mov_b32 s7, s4 +; GCN-NEXT: s_mov_b32 s8, s4 +; GCN-NEXT: s_mov_b32 s9, s4 +; GCN-NEXT: s_mov_b32 s10, s4 +; GCN-NEXT: s_mov_b32 s11, s4 +; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GCN-NEXT: v_mov_b32_e32 v2, v1 +; GCN-NEXT: v_mov_b32_e32 v3, v1 +; GCN-NEXT: v_mov_b32_e32 v5, s13 +; GCN-NEXT: image_store v[0:3], v[4:5], s[4:11] unorm +; GCN-NEXT: s_waitcnt vmcnt(0) +; GCN-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: main: +; GFX10: ; %bb.0: ; %bb +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s4, 0 +; GFX10-NEXT: v_mov_b32_e32 v1, 0 +; GFX10-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0 +; GFX10-NEXT: s_mov_b32 s10, s4 +; GFX10-NEXT: s_mov_b32 s11, s4 +; GFX10-NEXT: v_mov_b32_e32 v4, s10 +; GFX10-NEXT: v_mov_b32_e32 v2, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX10-NEXT: v_mov_b32_e32 v3, v1 +; GFX10-NEXT: v_mov_b32_e32 v5, s11 +; GFX10-NEXT: s_mov_b32 s5, s4 +; GFX10-NEXT: s_mov_b32 s6, s4 +; GFX10-NEXT: s_mov_b32 s7, s4 +; GFX10-NEXT: s_mov_b32 s8, s4 +; GFX10-NEXT: s_mov_b32 s9, s4 +; GFX10-NEXT: image_store v[0:3], v[4:5], s[4:11] dim:SQ_RSRC_IMG_2D unorm +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: main: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0 +; GFX11-NEXT: s_mov_b32 s6, s0 +; GFX11-NEXT: s_mov_b32 s7, s0 +; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, s6 +; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo +; GFX11-NEXT: v_mov_b32_e32 v5, s7 +; GFX11-NEXT: s_mov_b32 s1, s0 +; GFX11-NEXT: v_mov_b32_e32 v2, v1 +; GFX11-NEXT: v_mov_b32_e32 v3, v1 +; GFX11-NEXT: s_mov_b32 s2, s0 +; GFX11-NEXT: s_mov_b32 s3, s0 +; GFX11-NEXT: s_mov_b32 s4, s0 +; GFX11-NEXT: s_mov_b32 s5, s0 +; GFX11-NEXT: image_store v[0:3], v[4:5], s[0:7] dim:SQ_RSRC_IMG_2D unorm +; GFX11-NEXT: s_setpc_b64 s[30:31] +bb: + %i = bitcast <19 x i32> %arg to <38 x i16> + %i1 = extractelement <38 x i16> %i, i64 0 + %i2 = icmp eq i16 %i1, 0 + %i3 = zext i1 %i2 to i32 + %i4 = bitcast i32 %i3 to float + %i5 = insertelement <4 x float> zeroinitializer, float %i4, i64 0 + call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %i5, i32 0, i32 0, i32 0, <8 x i32> zeroinitializer, i32 0, i32 0) + ret void +} +declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX10PLUS: {{.*}} +; GPRIDX: {{.*}} +; MOVREL: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll index ac153183be64..1e1c90d142a1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll @@ -2626,6 +2626,132 @@ entry: ret double %ext } +define amdgpu_ps double @dyn_extract_v7f64_s_v_bitcast(<14 x float> inreg %userData, i32 %sel) { +; GCN-LABEL: dyn_extract_v7f64_s_v_bitcast: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: v_mov_b32_e32 v1, s2 +; GCN-NEXT: v_mov_b32_e32 v2, s3 +; GCN-NEXT: v_mov_b32_e32 v3, s4 +; GCN-NEXT: v_mov_b32_e32 v4, s5 +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GCN-NEXT: v_mov_b32_e32 v5, s6 +; GCN-NEXT: v_mov_b32_e32 v6, s7 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0 +; GCN-NEXT: v_mov_b32_e32 v7, s8 +; GCN-NEXT: v_mov_b32_e32 v8, s9 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0 +; GCN-NEXT: v_mov_b32_e32 v9, s10 +; GCN-NEXT: v_mov_b32_e32 v10, s11 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0 +; GCN-NEXT: v_mov_b32_e32 v11, s12 +; GCN-NEXT: v_mov_b32_e32 v12, s13 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0 +; GCN-NEXT: v_mov_b32_e32 v13, s14 +; GCN-NEXT: v_mov_b32_e32 v14, s15 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0 +; GCN-NEXT: v_cndmask_b32_e32 v1, v1, v13, vcc +; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v14, vcc +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0 +; GCN-NEXT: ; kill: def $vgpr15 killed $sgpr2 killed $exec +; GCN-NEXT: ; kill: def $vgpr16 killed $sgpr3 killed $exec +; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v15, vcc +; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v16, vcc +; GCN-NEXT: v_readfirstlane_b32 s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s1, v1 +; GCN-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: dyn_extract_v7f64_s_v_bitcast: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: v_mov_b32_e32 v2, s5 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX10-NEXT: s_mov_b32 s0, s14 +; GFX10-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, s3, v2, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s6, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s7, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s8, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s9, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s11, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s12, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s13, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s15, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, s2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v2, s3, vcc_lo +; GFX10-NEXT: v_readfirstlane_b32 s0, v0 +; GFX10-NEXT: v_readfirstlane_b32 s1, v1 +; GFX10-NEXT: ; return to shader part epilog +; +; GFX11-LABEL: dyn_extract_v7f64_s_v_bitcast: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s5 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX11-NEXT: s_mov_b32 s0, s14 +; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v2, s3, v2, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s6, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s7, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s8, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s9, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s10, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s11, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s12, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s13, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, s0, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, s15, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, s2, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e64 v1, v2, s3, vcc_lo +; GFX11-NEXT: v_readfirstlane_b32 s0, v0 +; GFX11-NEXT: v_readfirstlane_b32 s1, v1 +; GFX11-NEXT: ; return to shader part epilog +entry: + %bc = bitcast <14 x float> %userData to <7 x double> + %ext = extractelement <7 x double> %bc, i32 %sel + ret double %ext +} + +define amdgpu_ps i64 @dyn_extract_v7i64_s_v_bitcast(<14 x i32> inreg %userData, i32 %sel) { +; GCN-LABEL: dyn_extract_v7i64_s_v_bitcast: +; GCN: ; %bb.0: ; %entry +; GCN-NEXT: s_mov_b32 s0, s10 +; GCN-NEXT: s_mov_b32 s1, s11 +; GCN-NEXT: ; return to shader part epilog +; +; GFX10PLUS-LABEL: dyn_extract_v7i64_s_v_bitcast: +; GFX10PLUS: ; %bb.0: ; %entry +; GFX10PLUS-NEXT: s_mov_b32 s0, s10 +; GFX10PLUS-NEXT: s_mov_b32 s1, s11 +; GFX10PLUS-NEXT: ; return to shader part epilog +entry: + %.bc = bitcast <14 x i32> %userData to <7 x i64> + %ext = extractelement <7 x i64> %.bc, i32 4 + ret i64 %ext +} + define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel) { ; GCN-LABEL: dyn_extract_v7f64_s_v: ; GCN: ; %bb.0: ; %entry diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir index 10766b0f79d8..25652b69afa9 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir @@ -299,129 +299,6 @@ body: | S_NOP 0, implicit %12 ... --- -name: legal_v13s32 -body: | - bb.0: - liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12 - ; CHECK-LABEL: name: legal_v13s32 - ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4 - ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5 - ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6 - ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7 - ; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8 - ; CHECK-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9 - ; CHECK-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10 - ; CHECK-NEXT: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11 - ; CHECK-NEXT: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<13 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32) - ; CHECK-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<13 x s32>) - %0:_(s32) = COPY $vgpr0 - %1:_(s32) = COPY $vgpr1 - %2:_(s32) = COPY $vgpr2 - %3:_(s32) = COPY $vgpr3 - %4:_(s32) = COPY $vgpr4 - %5:_(s32) = COPY $vgpr5 - %6:_(s32) = COPY $vgpr6 - %7:_(s32) = COPY $vgpr7 - %8:_(s32) = COPY $vgpr8 - %9:_(s32) = COPY $vgpr9 - %10:_(s32) = COPY $vgpr10 - %11:_(s32) = COPY $vgpr11 - %12:_(s32) = COPY $vgpr12 - %13:_(<13 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12 - S_NOP 0, implicit %13 -... ---- -name: legal_v14s32 -body: | - bb.0: - liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13 - ; CHECK-LABEL: name: legal_v14s32 - ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4 - ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5 - ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6 - ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7 - ; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8 - ; CHECK-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9 - ; CHECK-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10 - ; CHECK-NEXT: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11 - ; CHECK-NEXT: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12 - ; CHECK-NEXT: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<14 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32) - ; CHECK-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<14 x s32>) - %0:_(s32) = COPY $vgpr0 - %1:_(s32) = COPY $vgpr1 - %2:_(s32) = COPY $vgpr2 - %3:_(s32) = COPY $vgpr3 - %4:_(s32) = COPY $vgpr4 - %5:_(s32) = COPY $vgpr5 - %6:_(s32) = COPY $vgpr6 - %7:_(s32) = COPY $vgpr7 - %8:_(s32) = COPY $vgpr8 - %9:_(s32) = COPY $vgpr9 - %10:_(s32) = COPY $vgpr10 - %11:_(s32) = COPY $vgpr11 - %12:_(s32) = COPY $vgpr12 - %13:_(s32) = COPY $vgpr13 - %14:_(<14 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13 - S_NOP 0, implicit %14 -... ---- -name: legal_v15s32 -body: | - bb.0: - liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14 - ; CHECK-LABEL: name: legal_v15s32 - ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3 - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4 - ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5 - ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6 - ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7 - ; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8 - ; CHECK-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9 - ; CHECK-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10 - ; CHECK-NEXT: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11 - ; CHECK-NEXT: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12 - ; CHECK-NEXT: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13 - ; CHECK-NEXT: [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14 - ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<15 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32) - ; CHECK-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<15 x s32>) - %0:_(s32) = COPY $vgpr0 - %1:_(s32) = COPY $vgpr1 - %2:_(s32) = COPY $vgpr2 - %3:_(s32) = COPY $vgpr3 - %4:_(s32) = COPY $vgpr4 - %5:_(s32) = COPY $vgpr5 - %6:_(s32) = COPY $vgpr6 - %7:_(s32) = COPY $vgpr7 - %8:_(s32) = COPY $vgpr8 - %9:_(s32) = COPY $vgpr9 - %10:_(s32) = COPY $vgpr10 - %11:_(s32) = COPY $vgpr11 - %12:_(s32) = COPY $vgpr12 - %13:_(s32) = COPY $vgpr13 - %14:_(s32) = COPY $vgpr14 - %15:_(<15 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14 - S_NOP 0, implicit %15 -... ---- name: legal_v16s32 body: | bb.0: -- GitLab From d7f59c8fb83cc00c58a826d542b81f98e7bd9526 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Tue, 13 Feb 2024 08:31:42 +0100 Subject: [PATCH 016/284] [mlir] Lower math dialect later in gpu-lower-to-nvvm-pipeline (#81489) This PR moves lowering of math dialect later in the pipeline. Because math dialect is lowered correctly by createConvertGpuOpsToNVVMOps for GPU target, and it needs to run it first. Reland #78556 --- .../GPU/Pipelines/GPUToNVVMPipeline.cpp | 2 +- mlir/test/Dialect/GPU/test-nvvm-pipeline.mlir | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 mlir/test/Dialect/GPU/test-nvvm-pipeline.mlir diff --git a/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp b/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp index 0b4739214bf2..935f0deaf9c8 100644 --- a/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp +++ b/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp @@ -51,7 +51,6 @@ void buildCommonPassPipeline( pm.addPass(createConvertVectorToSCFPass()); pm.addPass(createConvertSCFToCFPass()); pm.addPass(createConvertNVVMToLLVMPass()); - pm.addPass(createConvertMathToLLVMPass()); pm.addPass(createConvertFuncToLLVMPass()); pm.addPass(memref::createExpandStridedMetadataPass()); @@ -98,6 +97,7 @@ void buildHostPostPipeline(OpPassManager &pm, GpuModuleToBinaryPassOptions gpuModuleToBinaryPassOptions; gpuModuleToBinaryPassOptions.compilationTarget = options.cubinFormat; pm.addPass(createGpuModuleToBinaryPass(gpuModuleToBinaryPassOptions)); + pm.addPass(createConvertMathToLLVMPass()); pm.addPass(createCanonicalizerPass()); pm.addPass(createCSEPass()); pm.addPass(createReconcileUnrealizedCastsPass()); diff --git a/mlir/test/Dialect/GPU/test-nvvm-pipeline.mlir b/mlir/test/Dialect/GPU/test-nvvm-pipeline.mlir new file mode 100644 index 000000000000..07e719798b85 --- /dev/null +++ b/mlir/test/Dialect/GPU/test-nvvm-pipeline.mlir @@ -0,0 +1,30 @@ +// REQUIRES: host-supports-nvptx +// RUN: mlir-opt %s \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=isa" \ +// RUN: | FileCheck %s + +// RUN: mlir-opt %s \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=isa" \ +// RUN: --mlir-print-ir-after=convert-gpu-to-nvvm 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-NVVM + +// This test checks whether the GPU region is compiled correctly to PTX by +// pipeline. It doesn't test IR for GPU side, but it can test Host IR and +// generated PTX. + +// CHECK-LABEL: llvm.func @test_math(%arg0: f32) { +func.func @test_math(%arg0 : f32) { + %c2 = arith.constant 2 : index + %c1 = arith.constant 1 : index + // CHECK: gpu.launch_func @test_math_kernel::@test_math_kernel + // CHECK: gpu.binary @test_math_kernel [#gpu.object<#nvvm.target + gpu.launch + blocks(%0, %1, %2) in (%3 = %c1, %4 = %c1, %5 = %c1) + threads(%6, %7, %8) in (%9 = %c2, %10 = %c1, %11 = %c1) { + // CHECK-NVVM: __nv_expf + %s1 = math.exp %arg0 : f32 + gpu.printf "%f" %s1 : f32 + gpu.terminator + } + return +} \ No newline at end of file -- GitLab From d033366bd2189e33343ca93d276b40341dc39770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Tue, 13 Feb 2024 09:32:40 +0200 Subject: [PATCH 017/284] [LLD] [MinGW] Implement the --lto-emit-asm and -plugin-opt=emit-llvm options (#81475) These were implemented in the COFF linker in 3923e61b96cf90123762f0e0381504efaba2d77a and d12b99a4313816cf99e97cb5f579e2d51ba72b0b. This matches the corresponding options in the ELF linker. --- lld/MinGW/Driver.cpp | 4 ++++ lld/MinGW/Options.td | 5 +++++ lld/test/MinGW/driver.test | 7 +++++++ 3 files changed, 16 insertions(+) diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp index 290eecaacda7..efd643f9a322 100644 --- a/lld/MinGW/Driver.cpp +++ b/lld/MinGW/Driver.cpp @@ -451,6 +451,10 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, add("-lto-cs-profile-generate"); if (auto *arg = args.getLastArg(OPT_lto_cs_profile_file)) add("-lto-cs-profile-file:" + StringRef(arg->getValue())); + if (args.hasArg(OPT_plugin_opt_emit_llvm)) + add("-lldemit:llvm"); + if (args.hasArg(OPT_lto_emit_asm)) + add("-lldemit:asm"); if (auto *a = args.getLastArg(OPT_thinlto_cache_dir)) add("-lldltocache:" + StringRef(a->getValue())); diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td index 02f00f27406c..9a0a96aac7f1 100644 --- a/lld/MinGW/Options.td +++ b/lld/MinGW/Options.td @@ -158,6 +158,8 @@ def lto_cs_profile_generate: FF<"lto-cs-profile-generate">, HelpText<"Perform context sensitive PGO instrumentation">; def lto_cs_profile_file: JJ<"lto-cs-profile-file=">, HelpText<"Context sensitive profile file path">; +def lto_emit_asm: FF<"lto-emit-asm">, + HelpText<"Emit assembly code">; def thinlto_cache_dir: JJ<"thinlto-cache-dir=">, HelpText<"Path to ThinLTO cached object file directory">; @@ -181,6 +183,9 @@ def: J<"plugin-opt=cs-profile-path=">, Alias, HelpText<"Alias for --lto-cs-profile-file">; def plugin_opt_dwo_dir_eq: J<"plugin-opt=dwo_dir=">, HelpText<"Directory to store .dwo files when LTO and debug fission are used">; +def plugin_opt_emit_asm: F<"plugin-opt=emit-asm">, + Alias, HelpText<"Alias for --lto-emit-asm">; +def plugin_opt_emit_llvm: F<"plugin-opt=emit-llvm">; def: J<"plugin-opt=jobs=">, Alias, HelpText<"Alias for --thinlto-jobs=">; def plugin_opt_mcpu_eq: J<"plugin-opt=mcpu=">; diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test index 46b3b6d7a862..a4e9e5e1b19b 100644 --- a/lld/test/MinGW/driver.test +++ b/lld/test/MinGW/driver.test @@ -415,6 +415,13 @@ LTO_OPTS: -mllvm:-mcpu=x86-64 -opt:lldlto=2 -dwodir:foo -lto-cs-profile-generate RUN: ld.lld -### foo.o -m i386pep --lto-O2 --lto-CGO1 --lto-cs-profile-generate --lto-cs-profile-file=foo 2>&1 | FileCheck -check-prefix=LTO_OPTS2 %s LTO_OPTS2:-opt:lldlto=2 -opt:lldltocgo=1 -lto-cs-profile-generate -lto-cs-profile-file:foo +RUN: ld.lld -### foo.o -m i386pe -plugin-opt=emit-asm 2>&1 | FileCheck -check-prefix=LTO_EMIT_ASM %s +RUN: ld.lld -### foo.o -m i386pe --lto-emit-asm 2>&1 | FileCheck -check-prefix=LTO_EMIT_ASM %s +LTO_EMIT_ASM: -lldemit:asm + +RUN: ld.lld -### foo.o -m i386pe -plugin-opt=emit-llvm 2>&1 | FileCheck -check-prefix=LTO_EMIT_LLVM %s +LTO_EMIT_LLVM: -lldemit:llvm + Test GCC specific LTO options that GCC passes unconditionally, that we ignore. RUN: ld.lld -### foo.o -m i386pep -plugin /usr/lib/gcc/x86_64-w64-mingw32/10-posix/liblto_plugin.so -plugin-opt=/usr/lib/gcc/x86_64-w64-mingw32/10-posix/lto-wrapper -plugin-opt=-fresolution=/tmp/ccM9d4fP.res -plugin-opt=-pass-through=-lmingw32 2> /dev/null -- GitLab From 66f73100b8c758248724d53598165d850fdaf364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Tue, 13 Feb 2024 08:41:45 +0100 Subject: [PATCH 018/284] [clang][Interp] Fix ltor conversion for pointer types This special case is wrong, we need to handle pointer types here just like anything else. --- clang/lib/AST/Interp/Pointer.cpp | 6 +----- clang/test/AST/Interp/literals.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp index 8a0a15547b0f..3f85635f4367 100644 --- a/clang/lib/AST/Interp/Pointer.cpp +++ b/clang/lib/AST/Interp/Pointer.cpp @@ -232,11 +232,7 @@ std::optional Pointer::toRValue(const Context &Ctx) const { // Primitive values. if (std::optional T = Ctx.classify(Ty)) { - if (T == PT_Ptr || T == PT_FnPtr) { - R = Ptr.toAPValue(); - } else { - TYPE_SWITCH(*T, R = Ptr.deref().toAPValue()); - } + TYPE_SWITCH(*T, R = Ptr.deref().toAPValue()); return true; } diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp index f5b5f77ffc62..9202bb98c822 100644 --- a/clang/test/AST/Interp/literals.cpp +++ b/clang/test/AST/Interp/literals.cpp @@ -1105,3 +1105,13 @@ namespace NonConstReads { static_assert(z == 0, ""); // both-error {{not an integral constant expression}} \ // both-note {{read of non-const variable 'z'}} } + +/// This test passes a MaterializedTemporaryExpr to evaluateAsRValue. +/// That needs to return a null pointer after the lvalue-to-rvalue conversion. +/// We used to fail to do that. +namespace rdar8769025 { + __attribute__((nonnull)) void f1(int * const &p); + void test_f1() { + f1(0); // both-warning{{null passed to a callee that requires a non-null argument}} + } +} -- GitLab From d2ccf3393363f3bbcfd46e58a0ed2a6bd9170099 Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Tue, 13 Feb 2024 09:59:31 +0200 Subject: [PATCH 019/284] [libc++][sstream] Explicitly delete special member functions (#80254) The standard declares the copy constructors and copy assign operators as deleted. References: - https://eel.is/c++draft/string.streams --- libcxx/include/sstream | 16 ++++ .../istringstream/types.compile.pass.cpp | 75 +++++++++++++++++++ .../istringstream/types.pass.cpp | 39 ---------- .../ostringstream/types.compile.pass.cpp | 75 +++++++++++++++++++ .../ostringstream/types.pass.cpp | 39 ---------- .../stringbuf/types.compile.pass.cpp | 70 +++++++++++++++++ .../string.streams/stringbuf/types.pass.cpp | 39 ---------- .../stringstream/types.compile.pass.cpp | 72 ++++++++++++++++++ .../stringstream/types.pass.cpp | 39 ---------- 9 files changed, 308 insertions(+), 156 deletions(-) create mode 100644 libcxx/test/std/input.output/string.streams/istringstream/types.compile.pass.cpp delete mode 100644 libcxx/test/std/input.output/string.streams/istringstream/types.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/types.compile.pass.cpp delete mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/types.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringbuf/types.compile.pass.cpp delete mode 100644 libcxx/test/std/input.output/string.streams/stringbuf/types.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringstream/types.compile.pass.cpp delete mode 100644 libcxx/test/std/input.output/string.streams/stringstream/types.pass.cpp diff --git a/libcxx/include/sstream b/libcxx/include/sstream index 6c354cf0b397..8862e2ef99f8 100644 --- a/libcxx/include/sstream +++ b/libcxx/include/sstream @@ -48,10 +48,12 @@ public: template explicit basic_stringbuf(const basic_string& s, ios_base::openmode which = ios_base::in | ios_base::out); // C++20 + basic_stringbuf(const basic_stringbuf&) = delete; basic_stringbuf(basic_stringbuf&& rhs); basic_stringbuf(basic_stringbuf&& rhs, const allocator_type& a); // C++20 // [stringbuf.assign] Assign and swap: + basic_stringbuf& operator=(const basic_stringbuf&) = delete; basic_stringbuf& operator=(basic_stringbuf&& rhs); void swap(basic_stringbuf& rhs) noexcept(see below); // conditionally noexcept since C++20 @@ -119,9 +121,11 @@ public: template explicit basic_istringstream(const basic_string& s, ios_base::openmode which = ios_base::in); // C++20 + basic_istringstream(const basic_istringstream&) = delete; basic_istringstream(basic_istringstream&& rhs); // [istringstream.assign] Assign and swap: + basic_istringstream& operator=(const basic_istringstream&) = delete; basic_istringstream& operator=(basic_istringstream&& rhs); void swap(basic_istringstream& rhs); @@ -178,9 +182,11 @@ public: template explicit basic_ostringstream(const basic_string& s, ios_base::openmode which = ios_base::out); // C++20 + basic_ostringstream(const basic_ostringstream&) = delete; basic_ostringstream(basic_ostringstream&& rhs); // [ostringstream.assign] Assign and swap: + basic_ostringstream& operator=(const basic_ostringstream&) = delete; basic_ostringstream& operator=(basic_ostringstream&& rhs); void swap(basic_ostringstream& rhs); @@ -237,9 +243,11 @@ public: template explicit basic_stringstream(const basic_string& s, ios_base::openmode which = ios_base::out | ios_base::in); // C++20 + basic_stringstream(const basic_stringstream&) = delete; basic_stringstream(basic_stringstream&& rhs); // [stringstream.assign] Assign and swap: + basic_stringstream& operator=(const basic_stringstream&) = delete; basic_stringstream& operator=(basic_stringstream&& rhs); void swap(basic_stringstream& rhs); @@ -364,6 +372,7 @@ public: } #endif // _LIBCPP_STD_VER >= 20 + basic_stringbuf(const basic_stringbuf&) = delete; basic_stringbuf(basic_stringbuf&& __rhs) : __mode_(__rhs.__mode_) { __move_init(std::move(__rhs)); } #if _LIBCPP_STD_VER >= 20 @@ -374,6 +383,7 @@ public: #endif // [stringbuf.assign] Assign and swap: + basic_stringbuf& operator=(const basic_stringbuf&) = delete; basic_stringbuf& operator=(basic_stringbuf&& __rhs); void swap(basic_stringbuf& __rhs) #if _LIBCPP_STD_VER >= 20 @@ -822,12 +832,14 @@ public: : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::in) {} #endif // _LIBCPP_STD_VER >= 20 + basic_istringstream(const basic_istringstream&) = delete; _LIBCPP_HIDE_FROM_ABI basic_istringstream(basic_istringstream&& __rhs) : basic_istream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) { basic_istream<_CharT, _Traits>::set_rdbuf(&__sb_); } // [istringstream.assign] Assign and swap: + basic_istringstream& operator=(const basic_istringstream&) = delete; basic_istringstream& operator=(basic_istringstream&& __rhs) { basic_istream::operator=(std::move(__rhs)); __sb_ = std::move(__rhs.__sb_); @@ -929,12 +941,14 @@ public: : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::out) {} #endif // _LIBCPP_STD_VER >= 20 + basic_ostringstream(const basic_ostringstream&) = delete; _LIBCPP_HIDE_FROM_ABI basic_ostringstream(basic_ostringstream&& __rhs) : basic_ostream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) { basic_ostream<_CharT, _Traits>::set_rdbuf(&__sb_); } // [ostringstream.assign] Assign and swap: + basic_ostringstream& operator=(const basic_ostringstream&) = delete; basic_ostringstream& operator=(basic_ostringstream&& __rhs) { basic_ostream::operator=(std::move(__rhs)); __sb_ = std::move(__rhs.__sb_); @@ -1040,12 +1054,14 @@ public: : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch) {} #endif // _LIBCPP_STD_VER >= 20 + basic_stringstream(const basic_stringstream&) = delete; _LIBCPP_HIDE_FROM_ABI basic_stringstream(basic_stringstream&& __rhs) : basic_iostream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) { basic_istream<_CharT, _Traits>::set_rdbuf(&__sb_); } // [stringstream.assign] Assign and swap: + basic_stringstream& operator=(const basic_stringstream&) = delete; basic_stringstream& operator=(basic_stringstream&& __rhs) { basic_iostream::operator=(std::move(__rhs)); __sb_ = std::move(__rhs.__sb_); diff --git a/libcxx/test/std/input.output/string.streams/istringstream/types.compile.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/types.compile.pass.cpp new file mode 100644 index 000000000000..7be260ff79fa --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/istringstream/types.compile.pass.cpp @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// template , class Allocator = allocator > +// class basic_istringstream +// : public basic_istream +// { +// public: +// typedef charT char_type; +// typedef traits traits_type; +// typedef typename traits_type::int_type int_type; +// typedef typename traits_type::pos_type pos_type; +// typedef typename traits_type::off_type off_type; +// typedef Allocator allocator_type; +// +// basic_istringstream(const basic_istringstream&) = delete; +// basic_istringstream& operator=(const basic_istringstream&) = delete; +// +// basic_istringstream(basic_istringstream&& rhs); +// basic_istringstream& operator=(basic_istringstream&& rhs); + +#include +#include + +#include "test_macros.h" + +// Types + +static_assert(std::is_base_of, std::basic_istringstream >::value, ""); +static_assert(std::is_same::char_type, char>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_base_of, std::basic_istringstream >::value, ""); +static_assert(std::is_same::char_type, wchar_t>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, + ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, + ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, + ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); +#endif + +// Copy properties + +static_assert(!std::is_copy_constructible::value, ""); +static_assert(!std::is_copy_assignable::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(!std::is_copy_constructible::value, ""); +static_assert(!std::is_copy_assignable::value, ""); +#endif + +// Move properties + +static_assert(std::is_move_constructible::value, ""); +static_assert(std::is_move_assignable::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_move_constructible::value, ""); +static_assert(std::is_move_assignable::value, ""); +#endif diff --git a/libcxx/test/std/input.output/string.streams/istringstream/types.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/types.pass.cpp deleted file mode 100644 index da1b1de7358d..000000000000 --- a/libcxx/test/std/input.output/string.streams/istringstream/types.pass.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template , class Allocator = allocator > -// class basic_istringstream -// : public basic_istream -// { -// public: -// typedef charT char_type; -// typedef traits traits_type; -// typedef typename traits_type::int_type int_type; -// typedef typename traits_type::pos_type pos_type; -// typedef typename traits_type::off_type off_type; -// typedef Allocator allocator_type; - -#include -#include - -#include "test_macros.h" - -int main(int, char**) -{ - static_assert((std::is_base_of, std::basic_istringstream >::value), ""); - static_assert((std::is_same::char_type, char>::value), ""); - static_assert((std::is_same::traits_type, std::char_traits >::value), ""); - static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); - static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); - static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); - static_assert((std::is_same::allocator_type, std::allocator >::value), ""); - - return 0; -} diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/types.compile.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/types.compile.pass.cpp new file mode 100644 index 000000000000..df712bb0d097 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/ostringstream/types.compile.pass.cpp @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// template , class Allocator = allocator > +// class basic_ostringstream +// : public basic_ostream +// { +// public: +// typedef charT char_type; +// typedef traits traits_type; +// typedef typename traits_type::int_type int_type; +// typedef typename traits_type::pos_type pos_type; +// typedef typename traits_type::off_type off_type; +// typedef Allocator allocator_type; +// +// basic_ostringstream(const basic_ostringstream&) = delete; +// basic_ostringstream& operator=(const basic_ostringstream&) = delete; +// +// basic_ostringstream(basic_ostringstream&& rhs); +// basic_ostringstream& operator=(basic_ostringstream&& rhs); + +#include +#include + +#include "test_macros.h" + +// Types + +static_assert(std::is_base_of, std::basic_ostringstream >::value, ""); +static_assert(std::is_same::char_type, char>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_base_of, std::basic_ostringstream >::value, ""); +static_assert(std::is_same::char_type, wchar_t>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, + ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, + ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, + ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); +#endif + +// Copy properties + +static_assert(!std::is_copy_constructible::value, ""); +static_assert(!std::is_copy_assignable::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(!std::is_copy_constructible::value, ""); +static_assert(!std::is_copy_assignable::value, ""); +#endif + +// Move properties + +static_assert(std::is_move_constructible::value, ""); +static_assert(std::is_move_assignable::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_move_constructible::value, ""); +static_assert(std::is_move_assignable::value, ""); +#endif diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/types.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/types.pass.cpp deleted file mode 100644 index b29419279a37..000000000000 --- a/libcxx/test/std/input.output/string.streams/ostringstream/types.pass.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template , class Allocator = allocator > -// class basic_ostringstream -// : public basic_ostream -// { -// public: -// typedef charT char_type; -// typedef traits traits_type; -// typedef typename traits_type::int_type int_type; -// typedef typename traits_type::pos_type pos_type; -// typedef typename traits_type::off_type off_type; -// typedef Allocator allocator_type; - -#include -#include - -#include "test_macros.h" - -int main(int, char**) -{ - static_assert((std::is_base_of, std::basic_ostringstream >::value), ""); - static_assert((std::is_same::char_type, char>::value), ""); - static_assert((std::is_same::traits_type, std::char_traits >::value), ""); - static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); - static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); - static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); - static_assert((std::is_same::allocator_type, std::allocator >::value), ""); - - return 0; -} diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/types.compile.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/types.compile.pass.cpp new file mode 100644 index 000000000000..7743d3a73b7a --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringbuf/types.compile.pass.cpp @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// template , class Allocator = allocator > +// class basic_stringbuf +// : public basic_streambuf +// { +// public: +// typedef charT char_type; +// typedef traits traits_type; +// typedef typename traits_type::int_type int_type; +// typedef typename traits_type::pos_type pos_type; +// typedef typename traits_type::off_type off_type; +// typedef Allocator allocator_type; +// +// basic_stringbuf(const basic_stringbuf&) = delete; +// basic_stringbuf& operator=(const basic_stringbuf&) = delete; +// +// basic_stringbuf(basic_stringbuf&& rhs); +// basic_stringbuf& operator=(basic_stringbuf&& rhs); + +#include +#include + +#include "test_macros.h" + +static_assert(std::is_base_of, std::basic_stringbuf >::value, ""); +static_assert(std::is_same::char_type, char>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_base_of, std::basic_stringbuf >::value, ""); +static_assert(std::is_same::char_type, wchar_t>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); +#endif + +// Copy properties + +static_assert(!std::is_copy_constructible >::value, ""); +static_assert(!std::is_copy_assignable >::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(!std::is_copy_constructible >::value, ""); +static_assert(!std::is_copy_assignable >::value, ""); +#endif + +// Move properties + +static_assert(std::is_move_constructible >::value, ""); +static_assert(std::is_move_assignable >::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_move_constructible >::value, ""); +static_assert(std::is_move_assignable >::value, ""); +#endif diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/types.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/types.pass.cpp deleted file mode 100644 index cec616947a04..000000000000 --- a/libcxx/test/std/input.output/string.streams/stringbuf/types.pass.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template , class Allocator = allocator > -// class basic_stringbuf -// : public basic_streambuf -// { -// public: -// typedef charT char_type; -// typedef traits traits_type; -// typedef typename traits_type::int_type int_type; -// typedef typename traits_type::pos_type pos_type; -// typedef typename traits_type::off_type off_type; -// typedef Allocator allocator_type; - -#include -#include - -#include "test_macros.h" - -int main(int, char**) -{ - static_assert((std::is_base_of, std::basic_stringbuf >::value), ""); - static_assert((std::is_same::char_type, char>::value), ""); - static_assert((std::is_same::traits_type, std::char_traits >::value), ""); - static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); - static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); - static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); - static_assert((std::is_same::allocator_type, std::allocator >::value), ""); - - return 0; -} diff --git a/libcxx/test/std/input.output/string.streams/stringstream/types.compile.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/types.compile.pass.cpp new file mode 100644 index 000000000000..20486c85102a --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringstream/types.compile.pass.cpp @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// template , class Allocator = allocator > +// class basic_stringstream +// : public basic_iostream +// { +// public: +// typedef charT char_type; +// typedef traits traits_type; +// typedef typename traits_type::int_type int_type; +// typedef typename traits_type::pos_type pos_type; +// typedef typename traits_type::off_type off_type; +// typedef Allocator allocator_type; +// +// basic_stringstream(const basic_stringstream&) = delete; +// basic_stringstream& operator=(const basic_stringstream&) = delete; +// +// basic_stringstream(basic_stringstream&& rhs); +// basic_stringstream& operator=(basic_stringstream&& rhs); + +#include +#include + +#include "test_macros.h" + +// Types + +static_assert(std::is_base_of, std::basic_stringstream >::value, ""); +static_assert(std::is_same::char_type, char>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_base_of, std::basic_stringstream >::value, ""); +static_assert(std::is_same::char_type, wchar_t>::value, ""); +static_assert(std::is_same::traits_type, std::char_traits >::value, ""); +static_assert(std::is_same::int_type, std::char_traits::int_type>::value, ""); +static_assert(std::is_same::pos_type, std::char_traits::pos_type>::value, ""); +static_assert(std::is_same::off_type, std::char_traits::off_type>::value, ""); +static_assert(std::is_same::allocator_type, std::allocator >::value, ""); +#endif + +// Copy properties + +static_assert(!std::is_copy_constructible::value, ""); +static_assert(!std::is_copy_assignable::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(!std::is_copy_constructible::value, ""); +static_assert(!std::is_copy_assignable::value, ""); +#endif + +// Move properties + +static_assert(std::is_move_constructible::value, ""); +static_assert(std::is_move_assignable::value, ""); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::is_move_constructible::value, ""); +static_assert(std::is_move_assignable::value, ""); +#endif diff --git a/libcxx/test/std/input.output/string.streams/stringstream/types.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/types.pass.cpp deleted file mode 100644 index 11990e6a57f8..000000000000 --- a/libcxx/test/std/input.output/string.streams/stringstream/types.pass.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// template , class Allocator = allocator > -// class basic_stringstream -// : public basic_iostream -// { -// public: -// typedef charT char_type; -// typedef traits traits_type; -// typedef typename traits_type::int_type int_type; -// typedef typename traits_type::pos_type pos_type; -// typedef typename traits_type::off_type off_type; -// typedef Allocator allocator_type; - -#include -#include - -#include "test_macros.h" - -int main(int, char**) -{ - static_assert((std::is_base_of, std::basic_stringstream >::value), ""); - static_assert((std::is_same::char_type, char>::value), ""); - static_assert((std::is_same::traits_type, std::char_traits >::value), ""); - static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); - static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); - static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); - static_assert((std::is_same::allocator_type, std::allocator >::value), ""); - - return 0; -} -- GitLab From d30e941a03f7e70fb7875ede7b5d80342982e3a8 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 13 Feb 2024 08:06:16 +0000 Subject: [PATCH 020/284] [DAG] Add SelectionDAG::getShiftAmountConstant APInt variant (#81484) Asserts that the shift amount is in range and update ExpandShiftByConstant to use getShiftAmountConstant (and legal shift amount types). --- llvm/include/llvm/CodeGen/SelectionDAG.h | 2 + .../SelectionDAG/LegalizeIntegerTypes.cpp | 59 ++++++++++--------- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 ++ 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 886ec0b7940c..7bb12d8f065c 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -668,6 +668,8 @@ public: bool isTarget = false); SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes = true); + SDValue getShiftAmountConstant(const APInt &Val, EVT VT, const SDLoc &DL, + bool LegalTypes = true); SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget = false); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 39b7e0615541..e73a0921a46f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -2824,25 +2824,26 @@ void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, EVT NVT = InL.getValueType(); unsigned VTBits = N->getValueType(0).getSizeInBits(); unsigned NVTBits = NVT.getSizeInBits(); - EVT ShTy = N->getOperand(1).getValueType(); if (N->getOpcode() == ISD::SHL) { if (Amt.uge(VTBits)) { Lo = Hi = DAG.getConstant(0, DL, NVT); } else if (Amt.ugt(NVTBits)) { Lo = DAG.getConstant(0, DL, NVT); - Hi = DAG.getNode(ISD::SHL, DL, - NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); + Hi = DAG.getNode(ISD::SHL, DL, NVT, InL, + DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL)); } else if (Amt == NVTBits) { Lo = DAG.getConstant(0, DL, NVT); Hi = InL; } else { - Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); - Hi = DAG.getNode(ISD::OR, DL, NVT, - DAG.getNode(ISD::SHL, DL, NVT, InH, - DAG.getConstant(Amt, DL, ShTy)), - DAG.getNode(ISD::SRL, DL, NVT, InL, - DAG.getConstant(-Amt + NVTBits, DL, ShTy))); + Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, + DAG.getShiftAmountConstant(Amt, NVT, DL)); + Hi = DAG.getNode( + ISD::OR, DL, NVT, + DAG.getNode(ISD::SHL, DL, NVT, InH, + DAG.getShiftAmountConstant(Amt, NVT, DL)), + DAG.getNode(ISD::SRL, DL, NVT, InL, + DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL))); } return; } @@ -2851,19 +2852,21 @@ void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, if (Amt.uge(VTBits)) { Lo = Hi = DAG.getConstant(0, DL, NVT); } else if (Amt.ugt(NVTBits)) { - Lo = DAG.getNode(ISD::SRL, DL, - NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); + Lo = DAG.getNode(ISD::SRL, DL, NVT, InH, + DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL)); Hi = DAG.getConstant(0, DL, NVT); } else if (Amt == NVTBits) { Lo = InH; Hi = DAG.getConstant(0, DL, NVT); } else { - Lo = DAG.getNode(ISD::OR, DL, NVT, - DAG.getNode(ISD::SRL, DL, NVT, InL, - DAG.getConstant(Amt, DL, ShTy)), - DAG.getNode(ISD::SHL, DL, NVT, InH, - DAG.getConstant(-Amt + NVTBits, DL, ShTy))); - Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); + Lo = DAG.getNode( + ISD::OR, DL, NVT, + DAG.getNode(ISD::SRL, DL, NVT, InL, + DAG.getShiftAmountConstant(Amt, NVT, DL)), + DAG.getNode(ISD::SHL, DL, NVT, InH, + DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL))); + Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, + DAG.getShiftAmountConstant(Amt, NVT, DL)); } return; } @@ -2871,23 +2874,25 @@ void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); if (Amt.uge(VTBits)) { Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, - DAG.getConstant(NVTBits - 1, DL, ShTy)); + DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL)); } else if (Amt.ugt(NVTBits)) { Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, - DAG.getConstant(Amt - NVTBits, DL, ShTy)); + DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL)); Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, - DAG.getConstant(NVTBits - 1, DL, ShTy)); + DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL)); } else if (Amt == NVTBits) { Lo = InH; Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, - DAG.getConstant(NVTBits - 1, DL, ShTy)); + DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL)); } else { - Lo = DAG.getNode(ISD::OR, DL, NVT, - DAG.getNode(ISD::SRL, DL, NVT, InL, - DAG.getConstant(Amt, DL, ShTy)), - DAG.getNode(ISD::SHL, DL, NVT, InH, - DAG.getConstant(-Amt + NVTBits, DL, ShTy))); - Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); + Lo = DAG.getNode( + ISD::OR, DL, NVT, + DAG.getNode(ISD::SRL, DL, NVT, InL, + DAG.getShiftAmountConstant(Amt, NVT, DL)), + DAG.getNode(ISD::SHL, DL, NVT, InH, + DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL))); + Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, + DAG.getShiftAmountConstant(Amt, NVT, DL)); } } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 55eee780d512..421bb516ad24 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1734,6 +1734,12 @@ SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, return getConstant(Val, DL, ShiftVT); } +SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT, + const SDLoc &DL, bool LegalTypes) { + assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift"); + return getShiftAmountConstant(Val.getZExtValue(), VT, DL, LegalTypes); +} + SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget) { return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget); -- GitLab From a1efe56ace6099959ac97fcb09b9a7837b6d0255 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Tue, 13 Feb 2024 08:59:46 +0100 Subject: [PATCH 021/284] Remove an unused variable in release build. --- llvm/lib/Analysis/ValueTracking.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 220ef32c2d3c..92c9162a1f8f 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -4279,9 +4279,8 @@ static KnownFPClass computeKnownFPClassFromContext(const Value *V, if (!AssumeVH) continue; CallInst *I = cast(AssumeVH); - const Function *F = I->getFunction(); - assert(F == Q.CxtI->getParent()->getParent() && + assert(I->getFunction() == Q.CxtI->getParent()->getParent() && "Got assumption for the wrong function!"); assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume && "must be an assume intrinsic"); -- GitLab From 87d771193490e6604f132752438bd8404c83498c Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Tue, 13 Feb 2024 09:07:51 +0100 Subject: [PATCH 022/284] [AMDGPU][SIMemoryLegalizer] Fix order of GL0/1_INV on GFX10/11 (#81450) Fixes SWDEV-443292 --- llvm/docs/AMDGPUUsage.rst | 32 +- llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 5 +- .../AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll | 124 ++-- .../AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll | 128 ++--- .../memory-legalizer-atomic-fence.ll | 96 ++-- .../atomic_optimizations_global_pointer.ll | 88 +-- llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll | 2 +- llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll | 36 +- .../global-saddr-atomics-min-max-system.ll | 128 ++--- .../CodeGen/AMDGPU/global-saddr-atomics.ll | 248 ++++---- llvm/test/CodeGen/AMDGPU/global-saddr-load.ll | 16 +- .../CodeGen/AMDGPU/insert-waitcnts-crash.ll | 2 +- .../CodeGen/AMDGPU/memory-legalizer-fence.ll | 96 ++-- .../AMDGPU/memory-legalizer-flat-agent.ll | 544 +++++++++--------- .../AMDGPU/memory-legalizer-flat-system.ll | 544 +++++++++--------- .../AMDGPU/memory-legalizer-global-agent.ll | 544 +++++++++--------- .../AMDGPU/memory-legalizer-global-system.ll | 512 ++++++++--------- 17 files changed, 1574 insertions(+), 1571 deletions(-) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 2b4ca0a3c84e..8c126906fbfd 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -12291,8 +12291,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. before invalidating the caches. - 3. buffer_gl0_inv; - buffer_gl1_inv + 3. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following @@ -12321,8 +12321,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. before invalidating the caches. - 3. buffer_gl0_inv; - buffer_gl1_inv + 3. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following @@ -12428,8 +12428,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. invalidating the caches. - 3. buffer_gl0_inv; - buffer_gl1_inv + 3. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following @@ -12459,8 +12459,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. invalidating the caches. - 3. buffer_gl0_inv; - buffer_gl1_inv + 3. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following @@ -12655,8 +12655,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. the fence-paired-atomic. - 2. buffer_gl0_inv; - buffer_gl1_inv + 2. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following global/generic @@ -13369,8 +13369,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. invalidating the caches. - 4. buffer_gl0_inv; - buffer_gl1_inv + 4. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following @@ -13444,8 +13444,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. invalidating the caches. - 4. buffer_gl0_inv; - buffer_gl1_inv + 4. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following @@ -13672,8 +13672,8 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-gfx11-table`. requirements of release. - 2. buffer_gl0_inv; - buffer_gl1_inv + 2. buffer_gl1_inv; + buffer_gl0_inv - Must happen before any following diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp index 84b9330ef963..f62e808b33e4 100644 --- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp +++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -2030,8 +2030,11 @@ bool SIGfx10CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, switch (Scope) { case SIAtomicScope::SYSTEM: case SIAtomicScope::AGENT: - BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV)); + // The order of invalidates matter here. We must invalidate "outer in" + // so L1 -> L0 to avoid L0 pulling in stale data from L1 when it is + // invalidated. BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL1_INV)); + BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV)); Changed = true; break; case SIAtomicScope::WORKGROUP: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll index 25cee8724497..b04bc04ab226 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll @@ -338,8 +338,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -350,8 +350,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -416,8 +416,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %ou ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -428,8 +428,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %ou ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -495,8 +495,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_system(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v0, v1, v0, s[2:3] offset:16 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -507,8 +507,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_system(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v0, v1, v0, s[2:3] offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -563,8 +563,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) #1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i32: @@ -574,8 +574,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) #1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4 ret void @@ -629,8 +629,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) % ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i32_offset: @@ -640,8 +640,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) % ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1] offset:16 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4 @@ -696,8 +696,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_system(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v1, v0, s[0:1] offset:16 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i32_offset_system: @@ -707,8 +707,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_system(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v1, v0, s[0:1] offset:16 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4 @@ -780,8 +780,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v1, v0, v1, s[2:3] offset:20 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -792,8 +792,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v1, v0, v1, s[2:3] offset:20 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -861,8 +861,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec v0, v1, s[0:1] offset:20 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i32_offset_addr64: @@ -872,8 +872,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:20 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id @@ -937,8 +937,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 { ; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: flat_store_dword v[0:1], v2 @@ -952,8 +952,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 { ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: s_endpgm @@ -1023,8 +1023,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1 ; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: flat_store_dword v[0:1], v2 @@ -1038,8 +1038,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1 ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: s_endpgm @@ -1110,8 +1110,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr % ; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: flat_atomic_dec v2, v[0:1], v2 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: flat_store_dword v[0:1], v2 @@ -1125,8 +1125,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr % ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: s_endpgm @@ -1183,8 +1183,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_dec v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i32: @@ -1196,8 +1196,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4 ret void @@ -1256,8 +1256,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_dec v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i32_offset: @@ -1269,8 +1269,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:16 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr %ptr, i32 4 %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4 @@ -1330,8 +1330,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_dec v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i32_offset_system: @@ -1343,8 +1343,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:16 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr %ptr, i32 4 %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4 @@ -1430,8 +1430,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr % ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: flat_atomic_dec v3, v[0:1], v3 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 @@ -1450,8 +1450,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr % ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: flat_atomic_dec_u32 v3, v[0:1], v3 offset:20 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 @@ -1532,8 +1532,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_dec v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i32_offset_addr64: @@ -1549,8 +1549,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:20 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i32, ptr %ptr, i32 %id @@ -1628,8 +1628,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 { ; GFX10-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -1644,8 +1644,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 { ; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: s_endpgm @@ -1729,8 +1729,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1 ; GFX10-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -1745,8 +1745,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1 ; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[0:1] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: s_endpgm @@ -1807,8 +1807,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i64: @@ -1821,8 +1821,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8 ret void @@ -1885,8 +1885,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i64_offset: @@ -1899,8 +1899,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] offset:32 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr %ptr, i32 4 %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8 @@ -1964,8 +1964,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i64_offset_system: @@ -1978,8 +1978,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_dec_u64 v[2:3], v[0:1] offset:32 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr %ptr, i32 4 %result = atomicrmw udec_wrap ptr %gep, i64 42 seq_cst, align 8 @@ -2075,8 +2075,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr % ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 @@ -2097,8 +2097,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr % ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] offset:40 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 @@ -2183,8 +2183,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_dec_noret_i64_offset_addr64: @@ -2201,8 +2201,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] offset:40 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i64, ptr %ptr, i32 %id @@ -2651,8 +2651,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -2664,8 +2664,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2734,8 +2734,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %ou ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -2747,8 +2747,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %ou ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2818,8 +2818,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_system(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -2831,8 +2831,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_system(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2891,8 +2891,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) #1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i64: @@ -2903,8 +2903,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) #1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8 ret void @@ -2962,8 +2962,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) % ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:32 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i64_offset: @@ -2974,8 +2974,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) % ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:32 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8 @@ -3034,8 +3034,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_system(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:32 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i64_offset_system: @@ -3046,8 +3046,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_system(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:32 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8 @@ -3123,8 +3123,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -3136,8 +3136,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -3209,8 +3209,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_dec_x2 v0, v[1:2], s[0:1] offset:40 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_dec_noret_i64_offset_addr64: @@ -3221,8 +3221,8 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_dec_u64 v0, v[1:2], s[0:1] offset:40 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll index 5e013371d873..f6a997fb0fb0 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll @@ -338,8 +338,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -350,8 +350,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -416,8 +416,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %ou ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -428,8 +428,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %ou ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -495,8 +495,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_sistem(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v0, v1, v0, s[2:3] offset:16 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -507,8 +507,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_sistem(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, v0, s[2:3] offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -563,8 +563,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i32: @@ -574,8 +574,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4 ret void @@ -629,8 +629,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) % ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i32_offset: @@ -640,8 +640,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) % ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4 @@ -696,8 +696,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_system(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v1, v0, s[0:1] offset:16 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i32_offset_system: @@ -707,8 +707,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_system(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, s[0:1] offset:16 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4 @@ -780,8 +780,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v1, v0, v1, s[2:3] offset:20 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -792,8 +792,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v1, v0, v1, s[2:3] offset:20 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -861,8 +861,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc v0, v1, s[0:1] offset:20 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i32_offset_addr64: @@ -872,8 +872,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:20 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id @@ -1322,8 +1322,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -1335,8 +1335,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1405,8 +1405,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %ou ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -1418,8 +1418,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %ou ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1489,8 +1489,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_system(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -1502,8 +1502,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_system(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[2:3] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1562,8 +1562,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i64: @@ -1574,8 +1574,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8 ret void @@ -1633,8 +1633,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) % ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i64_offset: @@ -1645,8 +1645,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) % ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8 @@ -1705,8 +1705,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_system(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:32 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i64_offset_system: @@ -1717,8 +1717,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_system(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4 %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8 @@ -1794,8 +1794,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1] ; GFX10-NEXT: s_endpgm ; @@ -1807,8 +1807,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v[0:1], v3, v[1:2], s[2:3] offset:40 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1880,8 +1880,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_atomic_inc_x2 v0, v[1:2], s[0:1] offset:40 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_atomic_inc_noret_i64_offset_addr64: @@ -1892,8 +1892,8 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_inc_u64 v0, v[1:2], s[0:1] offset:40 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id @@ -1957,8 +1957,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 { ; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: flat_store_dword v[0:1], v2 @@ -1972,8 +1972,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 { ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: s_endpgm @@ -2043,8 +2043,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1 ; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: flat_store_dword v[0:1], v2 @@ -2058,8 +2058,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1 ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: s_endpgm @@ -2130,8 +2130,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr % ; GFX10-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-NEXT: flat_atomic_inc v2, v[0:1], v2 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: flat_store_dword v[0:1], v2 @@ -2145,8 +2145,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr % ; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-NEXT: s_endpgm @@ -2203,8 +2203,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_inc v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i32: @@ -2216,8 +2216,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4 ret void @@ -2276,8 +2276,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_inc v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i32_offset: @@ -2289,8 +2289,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr %ptr, i32 4 %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4 @@ -2350,8 +2350,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_inc v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_system: @@ -2363,8 +2363,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i32, ptr %ptr, i32 4 %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4 @@ -2450,8 +2450,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr % ; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX10-NEXT: flat_atomic_inc v3, v[0:1], v3 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 @@ -2470,8 +2470,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr % ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: flat_atomic_inc_u32 v3, v[0:1], v3 offset:20 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 @@ -2552,8 +2552,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_inc v[0:1], v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i32_offset_addr64: @@ -2569,8 +2569,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:20 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i32, ptr %ptr, i32 %id @@ -2744,8 +2744,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 { ; GFX10-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -2760,8 +2760,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 { ; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: s_endpgm @@ -2845,8 +2845,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1 ; GFX10-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -2861,8 +2861,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1 ; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: s_endpgm @@ -2947,8 +2947,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr % ; GFX10-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -2963,8 +2963,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr % ; GFX11-NEXT: v_mov_b32_e32 v3, s3 ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[0:1] offset:32 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: flat_store_b64 v[2:3], v[0:1] ; GFX11-NEXT: s_endpgm @@ -3025,8 +3025,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i64: @@ -3039,8 +3039,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8 ret void @@ -3103,8 +3103,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 { ; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i64_offset: @@ -3117,8 +3117,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 { ; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:32 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr %ptr, i32 4 %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8 @@ -3182,8 +3182,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_system: @@ -3196,8 +3196,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_inc_u64 v[2:3], v[0:1] offset:32 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %gep = getelementptr i64, ptr %ptr, i32 4 %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8 @@ -3293,8 +3293,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr % ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_mov_b32_e32 v3, s1 ; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 @@ -3315,8 +3315,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr % ; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] offset:40 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4 @@ -3401,8 +3401,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1 ; GFX10-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: flat_atomic_inc_noret_i64_offset_addr64: @@ -3419,8 +3419,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1 ; GFX11-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] offset:40 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %id = call i32 @llvm.amdgcn.workitem.id.x() %gep.tid = getelementptr i64, ptr %ptr, i32 %id diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/memory-legalizer-atomic-fence.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/memory-legalizer-atomic-fence.ll index 03cd8332bb3a..2727fdec035d 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/memory-legalizer-atomic-fence.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/memory-legalizer-atomic-fence.ll @@ -27,32 +27,32 @@ define amdgpu_kernel void @system_one_as_acquire() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 16240 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: system_one_as_acquire ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 16240 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: system_one_as_acquire ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 1015 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: system_one_as_acquire ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 1015 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("one-as") acquire @@ -115,32 +115,32 @@ define amdgpu_kernel void @system_one_as_acq_rel() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 16240 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: system_one_as_acq_rel ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 16240 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: system_one_as_acq_rel ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 1015 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: system_one_as_acq_rel ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 1015 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("one-as") acq_rel @@ -164,32 +164,32 @@ define amdgpu_kernel void @system_one_as_seq_cst() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 16240 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: system_one_as_seq_cst ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 16240 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: system_one_as_seq_cst ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 1015 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: system_one_as_seq_cst ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 1015 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("one-as") seq_cst @@ -329,32 +329,32 @@ define amdgpu_kernel void @agent_one_as_acquire() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 16240 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: agent_one_as_acquire ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 16240 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: agent_one_as_acquire ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 1015 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: agent_one_as_acquire ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 1015 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("agent-one-as") acquire @@ -417,32 +417,32 @@ define amdgpu_kernel void @agent_one_as_acq_rel() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 16240 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: agent_one_as_acq_rel ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 16240 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: agent_one_as_acq_rel ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 1015 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: agent_one_as_acq_rel ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 1015 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("agent-one-as") acq_rel @@ -466,32 +466,32 @@ define amdgpu_kernel void @agent_one_as_seq_cst() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 16240 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: agent_one_as_seq_cst ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 16240 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: agent_one_as_seq_cst ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 1015 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: agent_one_as_seq_cst ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 1015 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("agent-one-as") seq_cst @@ -769,32 +769,32 @@ define amdgpu_kernel void @system_acquire() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 112 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: system_acquire ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 112 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: system_acquire ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 7 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: system_acquire ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 7 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence acquire @@ -857,32 +857,32 @@ define amdgpu_kernel void @system_acq_rel() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 112 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: system_acq_rel ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 112 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: system_acq_rel ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 7 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: system_acq_rel ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 7 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence acq_rel @@ -906,32 +906,32 @@ define amdgpu_kernel void @system_seq_cst() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 112 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: system_seq_cst ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 112 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: system_seq_cst ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 7 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: system_seq_cst ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 7 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence seq_cst @@ -1071,32 +1071,32 @@ define amdgpu_kernel void @agent_acquire() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 112 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: agent_acquire ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 112 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: agent_acquire ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 7 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: agent_acquire ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 7 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("agent") acquire @@ -1159,32 +1159,32 @@ define amdgpu_kernel void @agent_acq_rel() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 112 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: agent_acq_rel ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 112 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: agent_acq_rel ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 7 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: agent_acq_rel ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 7 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("agent") acq_rel @@ -1208,32 +1208,32 @@ define amdgpu_kernel void @agent_seq_cst() { ; GFX10WGP: bb.0.entry: ; GFX10WGP-NEXT: S_WAITCNT_soft 112 ; GFX10WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10WGP-NEXT: S_ENDPGM 0 ; ; GFX10CU-LABEL: name: agent_seq_cst ; GFX10CU: bb.0.entry: ; GFX10CU-NEXT: S_WAITCNT_soft 112 ; GFX10CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX10CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX10CU-NEXT: S_ENDPGM 0 ; ; GFX11WGP-LABEL: name: agent_seq_cst ; GFX11WGP: bb.0.entry: ; GFX11WGP-NEXT: S_WAITCNT_soft 7 ; GFX11WGP-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11WGP-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11WGP-NEXT: S_ENDPGM 0 ; ; GFX11CU-LABEL: name: agent_seq_cst ; GFX11CU: bb.0.entry: ; GFX11CU-NEXT: S_WAITCNT_soft 7 ; GFX11CU-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0 - ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: BUFFER_GL1_INV implicit $exec + ; GFX11CU-NEXT: BUFFER_GL0_INV implicit $exec ; GFX11CU-NEXT: S_ENDPGM 0 entry: fence syncscope("agent") seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll index b5cccb6c4bb9..8ee0ee3b27ba 100644 --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll @@ -99,8 +99,8 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1064-NEXT: s_mov_b32 s9, s3 ; GFX1064-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB0_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] @@ -132,8 +132,8 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1032-NEXT: s_mov_b32 s9, s3 ; GFX1032-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB0_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 @@ -167,8 +167,8 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1164-NEXT: s_mov_b32 s9, s3 ; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB0_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -203,8 +203,8 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1132-NEXT: s_mov_b32 s9, s3 ; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB0_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -420,8 +420,8 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1064-NEXT: s_mov_b32 s13, s7 ; GFX1064-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB1_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1] @@ -455,8 +455,8 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1032-NEXT: s_mov_b32 s9, s7 ; GFX1032-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB1_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -492,8 +492,8 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1164-NEXT: s_mov_b32 s13, s7 ; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB1_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX1164-NEXT: v_readfirstlane_b32 s0, v1 @@ -530,8 +530,8 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1132-NEXT: s_mov_b32 s9, s7 ; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB1_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1 ; GFX1132-NEXT: v_readfirstlane_b32 s2, v1 @@ -767,8 +767,8 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1064-NEXT: s_mov_b32 s9, s3 ; GFX1064-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB2_4: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] @@ -812,8 +812,8 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1032-NEXT: s_mov_b32 s9, s3 ; GFX1032-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB2_4: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s5 @@ -861,8 +861,8 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1164-NEXT: s_mov_b32 s9, s3 ; GFX1164-NEXT: buffer_atomic_add_u32 v0, off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB2_4: ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -910,8 +910,8 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1132-NEXT: s_mov_b32 s9, s3 ; GFX1132-NEXT: buffer_atomic_add_u32 v0, off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB2_4: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s5 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -1128,8 +1128,8 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1064-NEXT: s_mov_b32 s9, s3 ; GFX1064-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB3_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] @@ -1163,8 +1163,8 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1032-NEXT: s_mov_b32 s9, s3 ; GFX1032-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB3_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 @@ -1200,8 +1200,8 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1164-NEXT: s_mov_b32 s9, s3 ; GFX1164-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB3_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -1237,8 +1237,8 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1132-NEXT: s_mov_b32 s9, s3 ; GFX1132-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB3_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -1488,8 +1488,8 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1064-NEXT: s_mov_b32 s9, s7 ; GFX1064-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB4_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1] @@ -1529,8 +1529,8 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1032-NEXT: s_mov_b32 s9, s7 ; GFX1032-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB4_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -1572,8 +1572,8 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1164-NEXT: s_mov_b32 s9, s7 ; GFX1164-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB4_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX1164-NEXT: v_readfirstlane_b32 s2, v0 @@ -1618,8 +1618,8 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1132-NEXT: s_mov_b32 s9, s7 ; GFX1132-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB4_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX1132-NEXT: v_readfirstlane_b32 s2, v0 @@ -1777,8 +1777,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX10-NEXT: s_mov_b32 s4, s0 ; GFX10-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_mov_b32 s5, s1 ; GFX10-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX10-NEXT: s_endpgm @@ -1797,8 +1797,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX11-NEXT: s_mov_b32 s4, s0 ; GFX11-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_mov_b32 s5, s1 ; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0 ; GFX11-NEXT: s_nop 0 @@ -1954,8 +1954,8 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1064-NEXT: s_mov_b32 s9, s3 ; GFX1064-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB6_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] @@ -1988,8 +1988,8 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1032-NEXT: s_mov_b32 s9, s3 ; GFX1032-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB6_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 @@ -2024,8 +2024,8 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1164-NEXT: s_mov_b32 s9, s3 ; GFX1164-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB6_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -2061,8 +2061,8 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1132-NEXT: s_mov_b32 s9, s3 ; GFX1132-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB6_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -2281,8 +2281,8 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1064-NEXT: s_mov_b32 s13, s7 ; GFX1064-NEXT: buffer_atomic_sub v1, off, s[12:15], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB7_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1] @@ -2317,8 +2317,8 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1032-NEXT: s_mov_b32 s9, s7 ; GFX1032-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB7_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -2355,8 +2355,8 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1164-NEXT: s_mov_b32 s13, s7 ; GFX1164-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB7_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -2394,8 +2394,8 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1132-NEXT: s_mov_b32 s9, s7 ; GFX1132-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB7_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -2634,8 +2634,8 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1064-NEXT: s_mov_b32 s9, s3 ; GFX1064-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB8_4: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] @@ -2679,8 +2679,8 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1032-NEXT: s_mov_b32 s9, s3 ; GFX1032-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB8_4: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s5 @@ -2728,8 +2728,8 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1164-NEXT: s_mov_b32 s9, s3 ; GFX1164-NEXT: buffer_atomic_sub_u32 v0, off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB8_4: ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -2777,8 +2777,8 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX1132-NEXT: s_mov_b32 s9, s3 ; GFX1132-NEXT: buffer_atomic_sub_u32 v0, off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB8_4: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s5 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -3034,8 +3034,8 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1064-NEXT: s_mov_b32 s9, s3 ; GFX1064-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB9_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5] @@ -3072,8 +3072,8 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1032-NEXT: s_mov_b32 s9, s3 ; GFX1032-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB9_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4 @@ -3112,8 +3112,8 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1164-NEXT: s_mov_b32 s9, s3 ; GFX1164-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB9_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -3152,8 +3152,8 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace ; GFX1132-NEXT: s_mov_b32 s9, s3 ; GFX1132-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB9_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -3415,8 +3415,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1064-NEXT: s_mov_b32 s9, s7 ; GFX1064-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc ; GFX1064-NEXT: s_waitcnt vmcnt(0) -; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: buffer_gl1_inv +; GFX1064-NEXT: buffer_gl0_inv ; GFX1064-NEXT: .LBB10_2: ; GFX1064-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1] @@ -3459,8 +3459,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1032-NEXT: s_mov_b32 s9, s7 ; GFX1032-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc ; GFX1032-NEXT: s_waitcnt vmcnt(0) -; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: buffer_gl1_inv +; GFX1032-NEXT: buffer_gl0_inv ; GFX1032-NEXT: .LBB10_2: ; GFX1032-NEXT: s_waitcnt_depctr 0xffe3 ; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s0 @@ -3505,8 +3505,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1164-NEXT: s_mov_b32 s9, s7 ; GFX1164-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc ; GFX1164-NEXT: s_waitcnt vmcnt(0) -; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: buffer_gl1_inv +; GFX1164-NEXT: buffer_gl0_inv ; GFX1164-NEXT: .LBB10_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) @@ -3553,8 +3553,8 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace( ; GFX1132-NEXT: s_mov_b32 s9, s7 ; GFX1132-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc ; GFX1132-NEXT: s_waitcnt vmcnt(0) -; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: buffer_gl1_inv +; GFX1132-NEXT: buffer_gl0_inv ; GFX1132-NEXT: .LBB10_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -3722,8 +3722,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX10-NEXT: s_mov_b32 s4, s0 ; GFX10-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_mov_b32 s5, s1 ; GFX10-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; GFX10-NEXT: s_endpgm @@ -3742,8 +3742,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace( ; GFX11-NEXT: s_mov_b32 s4, s0 ; GFX11-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_mov_b32 s5, s1 ; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0 ; GFX11-NEXT: s_nop 0 diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll index e64f8c23c061..19a1d2d9dbd1 100644 --- a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll +++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll @@ -75,8 +75,8 @@ define float @syncscope_system(ptr %addr, float %val) #0 { ; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX1100-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX1100-NEXT: buffer_gl0_inv ; GFX1100-NEXT: buffer_gl1_inv +; GFX1100-NEXT: buffer_gl0_inv ; GFX1100-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 ; GFX1100-NEXT: s_or_b32 s0, vcc_lo, s0 ; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1) diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll index 135285cca7f4..490167ee3635 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp.ll @@ -156,8 +156,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 ; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 ; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 @@ -196,8 +196,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 ; GFX11-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 ; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3 @@ -357,8 +357,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr ; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 ; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 @@ -390,8 +390,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: .LBB1_2: ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 @@ -506,8 +506,8 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) # ; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 @@ -531,8 +531,8 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) # ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: .LBB2_2: ; GFX11-NEXT: s_endpgm %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst @@ -638,8 +638,8 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %p ; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 @@ -663,8 +663,8 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %p ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: .LBB3_2: ; GFX11-NEXT: s_endpgm %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst @@ -806,8 +806,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %pt ; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 ; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 @@ -838,8 +838,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %pt ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: .LBB4_2: ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 @@ -1006,8 +1006,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %p ; GFX10-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 ; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 @@ -1046,8 +1046,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %p ; GFX11-NEXT: v_add_f32_e32 v4, v5, v2 ; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5 ; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3 @@ -1325,8 +1325,8 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %p ; GFX10-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2 @@ -1357,8 +1357,8 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %p ; GFX11-NEXT: v_add_f32_e32 v0, v1, v2 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1 ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2 @@ -1631,8 +1631,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p ; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1 ; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 ; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 @@ -1669,8 +1669,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %p ; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 ; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3 @@ -1819,8 +1819,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) % ; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1 ; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 ; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3 @@ -1857,8 +1857,8 @@ define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) % ; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2 ; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3 ; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3 diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll index a8475385f6fd..6b4a6381d954 100644 --- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll +++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics-min-max-system.ll @@ -48,8 +48,8 @@ define amdgpu_ps float @global_max_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -75,8 +75,8 @@ define amdgpu_ps float @global_max_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -131,8 +131,8 @@ define amdgpu_ps float @global_max_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX10-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -158,8 +158,8 @@ define amdgpu_ps float @global_max_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX11-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -211,8 +211,8 @@ define amdgpu_ps void @global_max_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -235,8 +235,8 @@ define amdgpu_ps void @global_max_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -286,8 +286,8 @@ define amdgpu_ps void @global_max_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -310,8 +310,8 @@ define amdgpu_ps void @global_max_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_max_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -372,8 +372,8 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -403,8 +403,8 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -467,8 +467,8 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -498,8 +498,8 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -558,8 +558,8 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -585,8 +585,8 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -642,8 +642,8 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -669,8 +669,8 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -730,8 +730,8 @@ define amdgpu_ps float @global_min_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -757,8 +757,8 @@ define amdgpu_ps float @global_min_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -813,8 +813,8 @@ define amdgpu_ps float @global_min_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX10-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -840,8 +840,8 @@ define amdgpu_ps float @global_min_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX11-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -893,8 +893,8 @@ define amdgpu_ps void @global_min_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -917,8 +917,8 @@ define amdgpu_ps void @global_min_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -968,8 +968,8 @@ define amdgpu_ps void @global_min_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -992,8 +992,8 @@ define amdgpu_ps void @global_min_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_min_i32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -1054,8 +1054,8 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -1085,8 +1085,8 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -1149,8 +1149,8 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -1180,8 +1180,8 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -1240,8 +1240,8 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -1267,8 +1267,8 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -1324,8 +1324,8 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -1351,8 +1351,8 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -1412,8 +1412,8 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -1439,8 +1439,8 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -1495,8 +1495,8 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -1522,8 +1522,8 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -1575,8 +1575,8 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn(ptr addrspace(1) inreg %sbase ; GFX10-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -1599,8 +1599,8 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn(ptr addrspace(1) inreg %sbase ; GFX11-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -1650,8 +1650,8 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -1674,8 +1674,8 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_max_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -1736,8 +1736,8 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn(ptr addrspace(1) inreg % ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -1767,8 +1767,8 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn(ptr addrspace(1) inreg % ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -1831,8 +1831,8 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn_neg128(ptr addrspace(1) ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -1862,8 +1862,8 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn_neg128(ptr addrspace(1) ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -1922,8 +1922,8 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -1949,8 +1949,8 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -2006,8 +2006,8 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -2033,8 +2033,8 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -2094,8 +2094,8 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -2121,8 +2121,8 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX11-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -2177,8 +2177,8 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -2204,8 +2204,8 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -2257,8 +2257,8 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn(ptr addrspace(1) inreg %sbase ; GFX10-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -2281,8 +2281,8 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn(ptr addrspace(1) inreg %sbase ; GFX11-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -2332,8 +2332,8 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], v[4:5], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX10-NEXT: v_mov_b32_e32 v5, v0 ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -2356,8 +2356,8 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_min_u32_e32 v4, v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], v[4:5], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5 ; GFX11-NEXT: v_mov_b32_e32 v5, v0 ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] @@ -2418,8 +2418,8 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn(ptr addrspace(1) inreg % ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -2449,8 +2449,8 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn(ptr addrspace(1) inreg % ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -2513,8 +2513,8 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn_neg128(ptr addrspace(1) ; GFX10-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX10-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX10-NEXT: s_andn2_b64 exec, exec, s[0:1] @@ -2544,8 +2544,8 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn_neg128(ptr addrspace(1) ; GFX11-NEXT: v_cndmask_b32_e32 v7, v1, v9, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[5:6], v[7:10], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[9:10] ; GFX11-NEXT: s_or_b64 s[0:1], vcc, s[0:1] ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) @@ -2604,8 +2604,8 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -2631,8 +2631,8 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 @@ -2688,8 +2688,8 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX10-NEXT: global_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX10-NEXT: v_mov_b32_e32 v6, v4 ; GFX10-NEXT: v_mov_b32_e32 v5, v3 @@ -2715,8 +2715,8 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX11-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc ; GFX11-NEXT: global_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6], off offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[5:6] ; GFX11-NEXT: v_mov_b32_e32 v6, v4 ; GFX11-NEXT: v_mov_b32_e32 v5, v3 diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll index 94d21bef94e7..a3c8bb141fd9 100644 --- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll +++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll @@ -17,16 +17,16 @@ define amdgpu_ps void @global_xchg_saddr_i32_nortn(ptr addrspace(1) inreg %sbase ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap v0, v1, s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_i32_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -47,16 +47,16 @@ define amdgpu_ps void @global_xchg_saddr_i32_nortn_offset_2047(ptr addrspace(1) ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap v0, v1, s[2:3] offset:2047 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_i32_nortn_offset_2047: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[2:3] offset:2047 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -78,16 +78,16 @@ define amdgpu_ps void @global_xchg_saddr_i32_nortn_offset_neg2048(ptr addrspace( ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap v0, v1, s[2:3] offset:-2048 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_i32_nortn_offset_neg2048: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[2:3] offset:-2048 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -108,16 +108,16 @@ define amdgpu_ps float @global_xchg_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap v0, v0, v1, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_i32_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -142,16 +142,16 @@ define amdgpu_ps float @global_xchg_saddr_i32_rtn_2048(ptr addrspace(1) inreg %s ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc, 0, v3, vcc ; GFX10-NEXT: global_atomic_swap v0, v[2:3], v1, off glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_i32_rtn_2048: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b32 v0, v0, v1, s[2:3] offset:2048 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -173,16 +173,16 @@ define amdgpu_ps float @global_xchg_saddr_i32_rtn_neg2048(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap v0, v0, v1, s[2:3] offset:-2048 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_i32_rtn_neg2048: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b32 v0, v0, v1, s[2:3] offset:-2048 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -222,8 +222,8 @@ define amdgpu_ps float @global_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, ; GFX10-NEXT: v_readfirstlane_b32 s1, v3 ; GFX10-NEXT: global_atomic_swap v0, v0, v1, s[0:1] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_uniform_ptr_in_vgprs_rtn: @@ -235,8 +235,8 @@ define amdgpu_ps float @global_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, ; GFX11-NEXT: v_readfirstlane_b32 s1, v3 ; GFX11-NEXT: global_atomic_swap_b32 v0, v0, v1, s[0:1] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds %zext.offset = zext i32 %voffset to i64 @@ -270,8 +270,8 @@ define amdgpu_ps float @global_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 ; GFX10-NEXT: v_readfirstlane_b32 s1, v3 ; GFX10-NEXT: global_atomic_swap v0, v0, v1, s[0:1] offset:42 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset: @@ -283,8 +283,8 @@ define amdgpu_ps float @global_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 ; GFX11-NEXT: v_readfirstlane_b32 s1, v3 ; GFX11-NEXT: global_atomic_swap_b32 v0, v0, v1, s[0:1] offset:42 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds %zext.offset = zext i32 %voffset to i64 @@ -319,8 +319,8 @@ define amdgpu_ps void @global_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset ; GFX10-NEXT: v_readfirstlane_b32 s1, v3 ; GFX10-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_uniform_ptr_in_vgprs_nortn: @@ -332,8 +332,8 @@ define amdgpu_ps void @global_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset ; GFX11-NEXT: v_readfirstlane_b32 s1, v3 ; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds %zext.offset = zext i32 %voffset to i64 @@ -366,8 +366,8 @@ define amdgpu_ps void @global_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i3 ; GFX10-NEXT: v_readfirstlane_b32 s1, v3 ; GFX10-NEXT: global_atomic_swap v0, v1, s[0:1] offset:42 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset: @@ -379,8 +379,8 @@ define amdgpu_ps void @global_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i3 ; GFX11-NEXT: v_readfirstlane_b32 s1, v3 ; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] offset:42 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds %zext.offset = zext i32 %voffset to i64 @@ -410,16 +410,16 @@ define amdgpu_ps <2 x float> @global_xchg_saddr_i64_rtn(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_i64_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -440,16 +440,16 @@ define amdgpu_ps <2 x float> @global_xchg_saddr_i64_rtn_neg128(ptr addrspace(1) ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xchg_saddr_i64_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -471,16 +471,16 @@ define amdgpu_ps void @global_xchg_saddr_i64_nortn(ptr addrspace(1) inreg %sbase ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap_x2 v0, v[1:2], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_i64_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b64 v0, v[1:2], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -500,16 +500,16 @@ define amdgpu_ps void @global_xchg_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_swap_x2 v0, v[1:2], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xchg_saddr_i64_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_swap_b64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -534,16 +534,16 @@ define amdgpu_ps float @global_add_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add v0, v0, v1, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_add_saddr_i32_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -564,16 +564,16 @@ define amdgpu_ps float @global_add_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add v0, v0, v1, s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_add_saddr_i32_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -595,16 +595,16 @@ define amdgpu_ps void @global_add_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add v0, v1, s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_add_saddr_i32_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -624,16 +624,16 @@ define amdgpu_ps void @global_add_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add v0, v1, s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_add_saddr_i32_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -654,16 +654,16 @@ define amdgpu_ps <2 x float> @global_add_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_add_saddr_i64_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -684,16 +684,16 @@ define amdgpu_ps <2 x float> @global_add_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_add_saddr_i64_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -715,16 +715,16 @@ define amdgpu_ps void @global_add_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add_x2 v0, v[1:2], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_add_saddr_i64_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u64 v0, v[1:2], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -744,16 +744,16 @@ define amdgpu_ps void @global_add_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_add_x2 v0, v[1:2], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_add_saddr_i64_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_add_u64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -778,16 +778,16 @@ define amdgpu_ps float @global_sub_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub v0, v0, v1, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_sub_saddr_i32_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -808,16 +808,16 @@ define amdgpu_ps float @global_sub_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub v0, v0, v1, s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_sub_saddr_i32_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -839,16 +839,16 @@ define amdgpu_ps void @global_sub_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub v0, v1, s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_sub_saddr_i32_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u32 v0, v1, s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -868,16 +868,16 @@ define amdgpu_ps void @global_sub_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub v0, v1, s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_sub_saddr_i32_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -898,16 +898,16 @@ define amdgpu_ps <2 x float> @global_sub_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_sub_saddr_i64_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -928,16 +928,16 @@ define amdgpu_ps <2 x float> @global_sub_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_sub_saddr_i64_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -959,16 +959,16 @@ define amdgpu_ps void @global_sub_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub_x2 v0, v[1:2], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_sub_saddr_i64_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u64 v0, v[1:2], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -988,16 +988,16 @@ define amdgpu_ps void @global_sub_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_sub_x2 v0, v[1:2], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_sub_saddr_i64_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_sub_u64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1022,16 +1022,16 @@ define amdgpu_ps float @global_and_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and v0, v0, v1, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_and_saddr_i32_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1052,16 +1052,16 @@ define amdgpu_ps float @global_and_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and v0, v0, v1, s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_and_saddr_i32_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1083,16 +1083,16 @@ define amdgpu_ps void @global_and_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and v0, v1, s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_and_saddr_i32_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b32 v0, v1, s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1112,16 +1112,16 @@ define amdgpu_ps void @global_and_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and v0, v1, s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_and_saddr_i32_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1142,16 +1142,16 @@ define amdgpu_ps <2 x float> @global_and_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_and_saddr_i64_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1172,16 +1172,16 @@ define amdgpu_ps <2 x float> @global_and_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_and_saddr_i64_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1203,16 +1203,16 @@ define amdgpu_ps void @global_and_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and_x2 v0, v[1:2], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_and_saddr_i64_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b64 v0, v[1:2], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1232,16 +1232,16 @@ define amdgpu_ps void @global_and_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_and_x2 v0, v[1:2], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_and_saddr_i64_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_and_b64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1266,16 +1266,16 @@ define amdgpu_ps float @global_or_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or v0, v0, v1, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_or_saddr_i32_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1296,16 +1296,16 @@ define amdgpu_ps float @global_or_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %s ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or v0, v0, v1, s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_or_saddr_i32_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1327,16 +1327,16 @@ define amdgpu_ps void @global_or_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or v0, v1, s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_or_saddr_i32_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b32 v0, v1, s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1356,16 +1356,16 @@ define amdgpu_ps void @global_or_saddr_i32_nortn_neg128(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or v0, v1, s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_or_saddr_i32_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1386,16 +1386,16 @@ define amdgpu_ps <2 x float> @global_or_saddr_i64_rtn(ptr addrspace(1) inreg %sb ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_or_saddr_i64_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1416,16 +1416,16 @@ define amdgpu_ps <2 x float> @global_or_saddr_i64_rtn_neg128(ptr addrspace(1) in ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_or_saddr_i64_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1447,16 +1447,16 @@ define amdgpu_ps void @global_or_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or_x2 v0, v[1:2], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_or_saddr_i64_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b64 v0, v[1:2], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1476,16 +1476,16 @@ define amdgpu_ps void @global_or_saddr_i64_nortn_neg128(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_or_x2 v0, v[1:2], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_or_saddr_i64_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_or_b64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1510,16 +1510,16 @@ define amdgpu_ps float @global_xor_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor v0, v0, v1, s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xor_saddr_i32_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b32 v0, v0, v1, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1540,16 +1540,16 @@ define amdgpu_ps float @global_xor_saddr_i32_rtn_neg128(ptr addrspace(1) inreg % ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor v0, v0, v1, s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xor_saddr_i32_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1571,16 +1571,16 @@ define amdgpu_ps void @global_xor_saddr_i32_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor v0, v1, s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xor_saddr_i32_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b32 v0, v1, s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1600,16 +1600,16 @@ define amdgpu_ps void @global_xor_saddr_i32_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor v0, v1, s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xor_saddr_i32_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b32 v0, v1, s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1630,16 +1630,16 @@ define amdgpu_ps <2 x float> @global_xor_saddr_i64_rtn(ptr addrspace(1) inreg %s ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor_x2 v[0:1], v0, v[1:2], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xor_saddr_i64_rtn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b64 v[0:1], v0, v[1:2], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1660,16 +1660,16 @@ define amdgpu_ps <2 x float> @global_xor_saddr_i64_rtn_neg128(ptr addrspace(1) i ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor_x2 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_xor_saddr_i64_rtn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b64 v[0:1], v0, v[1:2], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1691,16 +1691,16 @@ define amdgpu_ps void @global_xor_saddr_i64_nortn(ptr addrspace(1) inreg %sbase, ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor_x2 v0, v[1:2], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xor_saddr_i64_nortn: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b64 v0, v[1:2], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -1720,16 +1720,16 @@ define amdgpu_ps void @global_xor_saddr_i64_nortn_neg128(ptr addrspace(1) inreg ; GFX10: ; %bb.0: ; GFX10-NEXT: global_atomic_xor_x2 v0, v[1:2], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_xor_saddr_i64_nortn_neg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_atomic_xor_b64 v0, v[1:2], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2620,8 +2620,8 @@ define amdgpu_ps float @global_cmpxchg_saddr_i32_rtn(ptr addrspace(1) inreg %sba ; GFX10-NEXT: v_mov_b32_e32 v3, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v0, v[2:3], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_cmpxchg_saddr_i32_rtn: @@ -2629,8 +2629,8 @@ define amdgpu_ps float @global_cmpxchg_saddr_i32_rtn(ptr addrspace(1) inreg %sba ; GFX11-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2654,8 +2654,8 @@ define amdgpu_ps float @global_cmpxchg_saddr_i32_rtn_neg128(ptr addrspace(1) inr ; GFX10-NEXT: v_mov_b32_e32 v3, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v0, v[2:3], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_cmpxchg_saddr_i32_rtn_neg128: @@ -2663,8 +2663,8 @@ define amdgpu_ps float @global_cmpxchg_saddr_i32_rtn_neg128(ptr addrspace(1) inr ; GFX11-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2689,8 +2689,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i32_nortn(ptr addrspace(1) inreg %sb ; GFX10-NEXT: v_mov_b32_e32 v3, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_cmpxchg_saddr_i32_nortn: @@ -2698,8 +2698,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i32_nortn(ptr addrspace(1) inreg %sb ; GFX11-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2721,8 +2721,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i32_nortn_neg128(ptr addrspace(1) in ; GFX10-NEXT: v_mov_b32_e32 v3, v1 ; GFX10-NEXT: global_atomic_cmpswap v0, v[2:3], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_cmpxchg_saddr_i32_nortn_neg128: @@ -2730,8 +2730,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i32_nortn_neg128(ptr addrspace(1) in ; GFX11-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v[2:3], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2756,8 +2756,8 @@ define amdgpu_ps <2 x float> @global_cmpxchg_saddr_i64_rtn(ptr addrspace(1) inre ; GFX10-NEXT: v_mov_b32_e32 v5, v1 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[0:1], v0, v[3:6], s[2:3] glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_cmpxchg_saddr_i64_rtn: @@ -2766,8 +2766,8 @@ define amdgpu_ps <2 x float> @global_cmpxchg_saddr_i64_rtn(ptr addrspace(1) inre ; GFX11-NEXT: v_mov_b32_e32 v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[0:1], v0, v[3:6], s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2793,8 +2793,8 @@ define amdgpu_ps <2 x float> @global_cmpxchg_saddr_i64_rtn_neg128(ptr addrspace( ; GFX10-NEXT: v_mov_b32_e32 v5, v1 ; GFX10-NEXT: global_atomic_cmpswap_x2 v[0:1], v0, v[3:6], s[2:3] offset:-128 glc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: global_cmpxchg_saddr_i64_rtn_neg128: @@ -2803,8 +2803,8 @@ define amdgpu_ps <2 x float> @global_cmpxchg_saddr_i64_rtn_neg128(ptr addrspace( ; GFX11-NEXT: v_mov_b32_e32 v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b64 v[0:1], v0, v[3:6], s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2831,8 +2831,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i64_nortn(ptr addrspace(1) inreg %sb ; GFX10-NEXT: v_mov_b32_e32 v5, v1 ; GFX10-NEXT: global_atomic_cmpswap_x2 v0, v[3:6], s[2:3] ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_cmpxchg_saddr_i64_nortn: @@ -2841,8 +2841,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i64_nortn(ptr addrspace(1) inreg %sb ; GFX11-NEXT: v_mov_b32_e32 v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b64 v0, v[3:6], s[2:3] ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset @@ -2866,8 +2866,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i64_nortn_neg128(ptr addrspace(1) in ; GFX10-NEXT: v_mov_b32_e32 v5, v1 ; GFX10-NEXT: global_atomic_cmpswap_x2 v0, v[3:6], s[2:3] offset:-128 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: global_cmpxchg_saddr_i64_nortn_neg128: @@ -2876,8 +2876,8 @@ define amdgpu_ps void @global_cmpxchg_saddr_i64_nortn_neg128(ptr addrspace(1) in ; GFX11-NEXT: v_mov_b32_e32 v5, v1 ; GFX11-NEXT: global_atomic_cmpswap_b64 v0, v[3:6], s[2:3] offset:-128 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: s_endpgm %zext.offset = zext i32 %voffset to i64 %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll index 6d99485b91fe..d9cbbc11f9a7 100644 --- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll +++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll @@ -3575,16 +3575,16 @@ define amdgpu_ps float @atomic_global_load_saddr_i32(ptr addrspace(1) inreg %sba ; GFX10: ; %bb.0: ; GFX10-NEXT: global_load_dword v0, v0, s[2:3] glc dlc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: atomic_global_load_saddr_i32: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_load_b32 v0, v0, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: atomic_global_load_saddr_i32: @@ -3612,16 +3612,16 @@ define amdgpu_ps float @atomic_global_load_saddr_i32_immneg128(ptr addrspace(1) ; GFX10: ; %bb.0: ; GFX10-NEXT: global_load_dword v0, v0, s[2:3] offset:-128 glc dlc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: atomic_global_load_saddr_i32_immneg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_load_b32 v0, v0, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: atomic_global_load_saddr_i32_immneg128: @@ -3650,16 +3650,16 @@ define amdgpu_ps <2 x float> @atomic_global_load_saddr_i64(ptr addrspace(1) inre ; GFX10: ; %bb.0: ; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3] glc dlc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: atomic_global_load_saddr_i64: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_load_b64 v[0:1], v0, s[2:3] glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: atomic_global_load_saddr_i64: @@ -3687,16 +3687,16 @@ define amdgpu_ps <2 x float> @atomic_global_load_saddr_i64_immneg128(ptr addrspa ; GFX10: ; %bb.0: ; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3] offset:-128 glc dlc ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: buffer_gl1_inv +; GFX10-NEXT: buffer_gl0_inv ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: atomic_global_load_saddr_i64_immneg128: ; GFX11: ; %bb.0: ; GFX11-NEXT: global_load_b64 v[0:1], v0, s[2:3] offset:-128 glc ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: buffer_gl1_inv +; GFX11-NEXT: buffer_gl0_inv ; GFX11-NEXT: ; return to shader part epilog ; ; GFX12-LABEL: atomic_global_load_saddr_i64_immneg128: diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll index 833aba9b26af..9e336a714ca6 100644 --- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll +++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll @@ -23,8 +23,8 @@ define fastcc i32 @foo() { ; CHECK-NEXT: $sgpr17 = S_ADDC_U32 internal $sgpr17, target-flags(amdgpu-gotprel32-hi) @bar + 12, implicit-def $scc, implicit internal $scc ; CHECK-NEXT: } ; CHECK-NEXT: S_WAITCNT_VSCNT undef $sgpr_null, 0 - ; CHECK-NEXT: BUFFER_GL0_INV implicit $exec ; CHECK-NEXT: BUFFER_GL1_INV implicit $exec + ; CHECK-NEXT: BUFFER_GL0_INV implicit $exec ; CHECK-NEXT: renamable $sgpr16_sgpr17 = S_LOAD_DWORDX2_IMM killed renamable $sgpr16_sgpr17, 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; CHECK-NEXT: $vgpr40 = V_WRITELANE_B32 killed $sgpr30, 0, killed $vgpr40 ; CHECK-NEXT: $vgpr40 = V_WRITELANE_B32 killed $sgpr31, 1, killed $vgpr40 diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll index 89ccf97c3f4b..77962fadcacf 100644 --- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll @@ -1224,14 +1224,14 @@ define amdgpu_kernel void @agent_acquire_fence() { ; ; GFX10-WGP-LABEL: agent_acquire_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: agent_acquire_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: agent_acquire_fence: @@ -1260,14 +1260,14 @@ define amdgpu_kernel void @agent_acquire_fence() { ; ; GFX11-WGP-LABEL: agent_acquire_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: agent_acquire_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("agent") acquire @@ -1338,14 +1338,14 @@ define amdgpu_kernel void @agent_acq_rel_fence() { ; ; GFX10-WGP-LABEL: agent_acq_rel_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: agent_acq_rel_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: agent_acq_rel_fence: @@ -1376,14 +1376,14 @@ define amdgpu_kernel void @agent_acq_rel_fence() { ; ; GFX11-WGP-LABEL: agent_acq_rel_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: agent_acq_rel_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("agent") acq_rel @@ -1403,14 +1403,14 @@ define amdgpu_kernel void @agent_seq_cst_fence() { ; ; GFX10-WGP-LABEL: agent_seq_cst_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: agent_seq_cst_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: agent_seq_cst_fence: @@ -1441,14 +1441,14 @@ define amdgpu_kernel void @agent_seq_cst_fence() { ; ; GFX11-WGP-LABEL: agent_seq_cst_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: agent_seq_cst_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("agent") seq_cst @@ -1468,14 +1468,14 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() { ; ; GFX10-WGP-LABEL: agent_one_as_acquire_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: agent_one_as_acquire_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: agent_one_as_acquire_fence: @@ -1504,14 +1504,14 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() { ; ; GFX11-WGP-LABEL: agent_one_as_acquire_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: agent_one_as_acquire_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("agent-one-as") acquire @@ -1582,14 +1582,14 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() { ; ; GFX10-WGP-LABEL: agent_one_as_acq_rel_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: agent_one_as_acq_rel_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: agent_one_as_acq_rel_fence: @@ -1620,14 +1620,14 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() { ; ; GFX11-WGP-LABEL: agent_one_as_acq_rel_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: agent_one_as_acq_rel_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("agent-one-as") acq_rel @@ -1647,14 +1647,14 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() { ; ; GFX10-WGP-LABEL: agent_one_as_seq_cst_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: agent_one_as_seq_cst_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: agent_one_as_seq_cst_fence: @@ -1685,14 +1685,14 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() { ; ; GFX11-WGP-LABEL: agent_one_as_seq_cst_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: agent_one_as_seq_cst_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("agent-one-as") seq_cst @@ -1712,14 +1712,14 @@ define amdgpu_kernel void @system_acquire_fence() { ; ; GFX10-WGP-LABEL: system_acquire_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: system_acquire_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: system_acquire_fence: @@ -1750,14 +1750,14 @@ define amdgpu_kernel void @system_acquire_fence() { ; ; GFX11-WGP-LABEL: system_acquire_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: system_acquire_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence acquire @@ -1830,14 +1830,14 @@ define amdgpu_kernel void @system_acq_rel_fence() { ; ; GFX10-WGP-LABEL: system_acq_rel_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: system_acq_rel_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: system_acq_rel_fence: @@ -1872,14 +1872,14 @@ define amdgpu_kernel void @system_acq_rel_fence() { ; ; GFX11-WGP-LABEL: system_acq_rel_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: system_acq_rel_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence acq_rel @@ -1899,14 +1899,14 @@ define amdgpu_kernel void @system_seq_cst_fence() { ; ; GFX10-WGP-LABEL: system_seq_cst_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: system_seq_cst_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: system_seq_cst_fence: @@ -1941,14 +1941,14 @@ define amdgpu_kernel void @system_seq_cst_fence() { ; ; GFX11-WGP-LABEL: system_seq_cst_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: system_seq_cst_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence seq_cst @@ -1968,14 +1968,14 @@ define amdgpu_kernel void @system_one_as_acquire_fence() { ; ; GFX10-WGP-LABEL: system_one_as_acquire_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: system_one_as_acquire_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: system_one_as_acquire_fence: @@ -2006,14 +2006,14 @@ define amdgpu_kernel void @system_one_as_acquire_fence() { ; ; GFX11-WGP-LABEL: system_one_as_acquire_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: system_one_as_acquire_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("one-as") acquire @@ -2086,14 +2086,14 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() { ; ; GFX10-WGP-LABEL: system_one_as_acq_rel_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: system_one_as_acq_rel_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: system_one_as_acq_rel_fence: @@ -2128,14 +2128,14 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() { ; ; GFX11-WGP-LABEL: system_one_as_acq_rel_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: system_one_as_acq_rel_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("one-as") acq_rel @@ -2155,14 +2155,14 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() { ; ; GFX10-WGP-LABEL: system_one_as_seq_cst_fence: ; GFX10-WGP: ; %bb.0: ; %entry -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: system_one_as_seq_cst_fence: ; GFX10-CU: ; %bb.0: ; %entry -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: system_one_as_seq_cst_fence: @@ -2197,14 +2197,14 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() { ; ; GFX11-WGP-LABEL: system_one_as_seq_cst_fence: ; GFX11-WGP: ; %bb.0: ; %entry -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: system_one_as_seq_cst_fence: ; GFX11-CU: ; %bb.0: ; %entry -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm entry: fence syncscope("one-as") seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll index d6c759f733b4..d9f5e64eae92 100644 --- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll @@ -299,8 +299,8 @@ define amdgpu_kernel void @flat_agent_acquire_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -314,8 +314,8 @@ define amdgpu_kernel void @flat_agent_acquire_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -397,8 +397,8 @@ define amdgpu_kernel void @flat_agent_acquire_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -410,8 +410,8 @@ define amdgpu_kernel void @flat_agent_acquire_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -445,8 +445,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -460,8 +460,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -543,8 +543,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -556,8 +556,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -1168,8 +1168,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw( ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw: @@ -1184,8 +1184,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw( ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw: @@ -1259,8 +1259,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw( ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acquire_atomicrmw: @@ -1274,8 +1274,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw( ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -1425,8 +1425,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw: @@ -1441,8 +1441,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw( ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw: @@ -1518,8 +1518,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acq_rel_atomicrmw: @@ -1533,8 +1533,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw( ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -1568,8 +1568,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw: @@ -1584,8 +1584,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw( ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw: @@ -1661,8 +1661,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_seq_cst_atomicrmw: @@ -1676,8 +1676,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw( ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -1711,8 +1711,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm ; @@ -1727,8 +1727,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm ; @@ -1807,8 +1807,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -1822,8 +1822,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { @@ -1859,8 +1859,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm ; @@ -1875,8 +1875,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm ; @@ -1957,8 +1957,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -1972,8 +1972,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { @@ -2009,8 +2009,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm ; @@ -2025,8 +2025,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm ; @@ -2107,8 +2107,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -2122,8 +2122,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { @@ -2275,8 +2275,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg: @@ -2292,8 +2292,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg: @@ -2363,8 +2363,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg: @@ -2376,8 +2376,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2530,8 +2530,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg: @@ -2547,8 +2547,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg: @@ -2620,8 +2620,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg: @@ -2633,8 +2633,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2672,8 +2672,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg: @@ -2689,8 +2689,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg: @@ -2762,8 +2762,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg: @@ -2775,8 +2775,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2814,8 +2814,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg: @@ -2831,8 +2831,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_cmpxchg: @@ -2902,8 +2902,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg: @@ -2915,8 +2915,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2954,8 +2954,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg: @@ -2971,8 +2971,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg: @@ -3042,8 +3042,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acquire_acquire_cmpxchg: @@ -3055,8 +3055,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3094,8 +3094,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg: @@ -3111,8 +3111,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg: @@ -3184,8 +3184,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_release_acquire_cmpxchg: @@ -3197,8 +3197,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3236,8 +3236,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg: @@ -3253,8 +3253,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg: @@ -3326,8 +3326,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg: @@ -3339,8 +3339,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3378,8 +3378,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg: @@ -3395,8 +3395,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg: @@ -3468,8 +3468,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg: @@ -3481,8 +3481,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3520,8 +3520,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg: @@ -3537,8 +3537,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_cmpxchg: @@ -3610,8 +3610,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg: @@ -3623,8 +3623,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3662,8 +3662,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg: @@ -3679,8 +3679,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_cmpxchg: @@ -3752,8 +3752,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg: @@ -3765,8 +3765,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3804,8 +3804,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg: @@ -3821,8 +3821,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_cmpxchg: @@ -3894,8 +3894,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_release_seq_cst_cmpxchg: @@ -3907,8 +3907,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3946,8 +3946,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg: @@ -3963,8 +3963,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg: @@ -4036,8 +4036,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg: @@ -4049,8 +4049,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -4088,8 +4088,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg: @@ -4105,8 +4105,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg: @@ -4178,8 +4178,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg: @@ -4191,8 +4191,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -4375,8 +4375,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -4394,8 +4394,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -4474,8 +4474,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -4487,8 +4487,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -4676,8 +4676,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -4695,8 +4695,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -4777,8 +4777,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -4790,8 +4790,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -4834,8 +4834,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -4853,8 +4853,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -4935,8 +4935,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -4948,8 +4948,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -4992,8 +4992,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5011,8 +5011,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5091,8 +5091,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5104,8 +5104,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5148,8 +5148,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5167,8 +5167,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5247,8 +5247,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5260,8 +5260,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5304,8 +5304,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5323,8 +5323,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5405,8 +5405,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5418,8 +5418,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5462,8 +5462,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5481,8 +5481,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5563,8 +5563,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5576,8 +5576,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5620,8 +5620,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5639,8 +5639,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5721,8 +5721,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5734,8 +5734,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5778,8 +5778,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5797,8 +5797,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5879,8 +5879,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5892,8 +5892,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5936,8 +5936,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5955,8 +5955,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6037,8 +6037,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6050,8 +6050,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6094,8 +6094,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6113,8 +6113,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6195,8 +6195,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6208,8 +6208,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6252,8 +6252,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6271,8 +6271,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6353,8 +6353,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6366,8 +6366,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6410,8 +6410,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6429,8 +6429,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6511,8 +6511,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6524,8 +6524,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6827,8 +6827,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -6843,8 +6843,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -6930,8 +6930,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 @@ -6944,8 +6944,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 @@ -6981,8 +6981,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -6997,8 +6997,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -7084,8 +7084,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 @@ -7098,8 +7098,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 @@ -7710,8 +7710,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw: @@ -7725,8 +7725,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw: @@ -7799,8 +7799,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acquire_atomicrmw: @@ -7813,8 +7813,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -7963,8 +7963,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw: @@ -7978,8 +7978,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw: @@ -8054,8 +8054,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw: @@ -8068,8 +8068,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -8102,8 +8102,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw: @@ -8117,8 +8117,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw: @@ -8193,8 +8193,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw: @@ -8207,8 +8207,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -8243,8 +8243,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm @@ -8260,8 +8260,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm @@ -8343,8 +8343,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -8359,8 +8359,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -8398,8 +8398,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm @@ -8415,8 +8415,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm @@ -8500,8 +8500,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -8516,8 +8516,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -8555,8 +8555,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm @@ -8572,8 +8572,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm @@ -8657,8 +8657,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -8673,8 +8673,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -8826,8 +8826,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg: @@ -8842,8 +8842,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg: @@ -8912,8 +8912,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg: @@ -8924,8 +8924,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9077,8 +9077,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg: @@ -9093,8 +9093,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg: @@ -9165,8 +9165,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg: @@ -9177,8 +9177,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9215,8 +9215,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg: @@ -9231,8 +9231,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg: @@ -9303,8 +9303,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg: @@ -9315,8 +9315,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9353,8 +9353,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg: @@ -9369,8 +9369,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg: @@ -9439,8 +9439,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg: @@ -9451,8 +9451,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9489,8 +9489,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg: @@ -9505,8 +9505,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg: @@ -9575,8 +9575,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg: @@ -9587,8 +9587,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9625,8 +9625,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg: @@ -9641,8 +9641,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg: @@ -9713,8 +9713,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg: @@ -9725,8 +9725,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9763,8 +9763,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg: @@ -9779,8 +9779,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg: @@ -9851,8 +9851,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg: @@ -9863,8 +9863,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9901,8 +9901,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg: @@ -9917,8 +9917,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg: @@ -9989,8 +9989,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg: @@ -10001,8 +10001,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10039,8 +10039,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg: @@ -10055,8 +10055,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg: @@ -10127,8 +10127,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg: @@ -10139,8 +10139,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10177,8 +10177,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg: @@ -10193,8 +10193,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg: @@ -10265,8 +10265,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg: @@ -10277,8 +10277,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10315,8 +10315,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg: @@ -10331,8 +10331,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg: @@ -10403,8 +10403,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg: @@ -10415,8 +10415,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10453,8 +10453,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg: @@ -10469,8 +10469,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg: @@ -10541,8 +10541,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg: @@ -10553,8 +10553,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10591,8 +10591,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg: @@ -10607,8 +10607,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg: @@ -10679,8 +10679,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg: @@ -10691,8 +10691,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10876,8 +10876,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -10896,8 +10896,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -10980,8 +10980,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -10994,8 +10994,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11185,8 +11185,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11205,8 +11205,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11291,8 +11291,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11305,8 +11305,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11351,8 +11351,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11371,8 +11371,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11457,8 +11457,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11471,8 +11471,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11517,8 +11517,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11537,8 +11537,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11621,8 +11621,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11635,8 +11635,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11681,8 +11681,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11701,8 +11701,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11785,8 +11785,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11799,8 +11799,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11845,8 +11845,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11865,8 +11865,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11951,8 +11951,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11965,8 +11965,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12011,8 +12011,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12031,8 +12031,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12117,8 +12117,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12131,8 +12131,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12177,8 +12177,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12197,8 +12197,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12283,8 +12283,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12297,8 +12297,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12343,8 +12343,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12363,8 +12363,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12449,8 +12449,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12463,8 +12463,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12509,8 +12509,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12529,8 +12529,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12615,8 +12615,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12629,8 +12629,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12675,8 +12675,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12695,8 +12695,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12781,8 +12781,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12795,8 +12795,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12841,8 +12841,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12861,8 +12861,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12947,8 +12947,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12961,8 +12961,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -13007,8 +13007,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -13027,8 +13027,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -13113,8 +13113,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -13127,8 +13127,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll index 461fe8b1296a..372f9ada5839 100644 --- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll @@ -299,8 +299,8 @@ define amdgpu_kernel void @flat_system_acquire_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -314,8 +314,8 @@ define amdgpu_kernel void @flat_system_acquire_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -399,8 +399,8 @@ define amdgpu_kernel void @flat_system_acquire_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -412,8 +412,8 @@ define amdgpu_kernel void @flat_system_acquire_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -447,8 +447,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -462,8 +462,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -547,8 +547,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -560,8 +560,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -1176,8 +1176,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw( ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acquire_atomicrmw: @@ -1192,8 +1192,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw( ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acquire_atomicrmw: @@ -1269,8 +1269,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw( ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acquire_atomicrmw: @@ -1284,8 +1284,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw( ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -1437,8 +1437,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw: @@ -1453,8 +1453,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw( ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_atomicrmw: @@ -1534,8 +1534,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acq_rel_atomicrmw: @@ -1549,8 +1549,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw( ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -1584,8 +1584,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw: @@ -1600,8 +1600,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw( ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_atomicrmw: @@ -1681,8 +1681,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_seq_cst_atomicrmw: @@ -1696,8 +1696,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw( ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -1731,8 +1731,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm ; @@ -1747,8 +1747,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm ; @@ -1829,8 +1829,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -1844,8 +1844,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { @@ -1881,8 +1881,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm ; @@ -1897,8 +1897,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm ; @@ -1983,8 +1983,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -1998,8 +1998,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { @@ -2035,8 +2035,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm ; @@ -2051,8 +2051,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm ; @@ -2137,8 +2137,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -2152,8 +2152,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { @@ -2305,8 +2305,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg: @@ -2322,8 +2322,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_cmpxchg: @@ -2395,8 +2395,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acquire_monotonic_cmpxchg: @@ -2408,8 +2408,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2564,8 +2564,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg: @@ -2581,8 +2581,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_cmpxchg: @@ -2658,8 +2658,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg: @@ -2671,8 +2671,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2710,8 +2710,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg: @@ -2727,8 +2727,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_cmpxchg: @@ -2804,8 +2804,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg: @@ -2817,8 +2817,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2856,8 +2856,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_monotonic_acquire_cmpxchg: @@ -2873,8 +2873,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_cmpxchg: @@ -2946,8 +2946,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_monotonic_acquire_cmpxchg: @@ -2959,8 +2959,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -2998,8 +2998,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg: @@ -3015,8 +3015,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_cmpxchg: @@ -3088,8 +3088,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acquire_acquire_cmpxchg: @@ -3101,8 +3101,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3140,8 +3140,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg: @@ -3157,8 +3157,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_cmpxchg: @@ -3234,8 +3234,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_release_acquire_cmpxchg: @@ -3247,8 +3247,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3286,8 +3286,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg: @@ -3303,8 +3303,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_cmpxchg: @@ -3380,8 +3380,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg: @@ -3393,8 +3393,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3432,8 +3432,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg: @@ -3449,8 +3449,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_cmpxchg: @@ -3526,8 +3526,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg: @@ -3539,8 +3539,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3578,8 +3578,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg: @@ -3595,8 +3595,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_cmpxchg: @@ -3672,8 +3672,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg: @@ -3685,8 +3685,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3724,8 +3724,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg: @@ -3741,8 +3741,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_cmpxchg: @@ -3818,8 +3818,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg: @@ -3831,8 +3831,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -3870,8 +3870,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_release_seq_cst_cmpxchg: @@ -3887,8 +3887,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_cmpxchg: @@ -3964,8 +3964,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_release_seq_cst_cmpxchg: @@ -3977,8 +3977,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -4016,8 +4016,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg: @@ -4033,8 +4033,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_cmpxchg: @@ -4110,8 +4110,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg: @@ -4123,8 +4123,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -4162,8 +4162,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg: @@ -4179,8 +4179,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_cmpxchg: @@ -4256,8 +4256,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg: @@ -4269,8 +4269,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -4453,8 +4453,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -4472,8 +4472,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -4554,8 +4554,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -4567,8 +4567,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -4758,8 +4758,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -4777,8 +4777,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -4863,8 +4863,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -4876,8 +4876,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -4920,8 +4920,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -4939,8 +4939,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5025,8 +5025,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5038,8 +5038,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5082,8 +5082,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5101,8 +5101,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5183,8 +5183,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5196,8 +5196,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5240,8 +5240,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5259,8 +5259,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5341,8 +5341,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5354,8 +5354,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5398,8 +5398,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5417,8 +5417,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5503,8 +5503,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5516,8 +5516,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5560,8 +5560,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5579,8 +5579,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5665,8 +5665,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5678,8 +5678,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5722,8 +5722,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5741,8 +5741,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5827,8 +5827,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -5840,8 +5840,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -5884,8 +5884,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -5903,8 +5903,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -5989,8 +5989,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6002,8 +6002,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6046,8 +6046,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6065,8 +6065,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6151,8 +6151,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6164,8 +6164,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6208,8 +6208,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6227,8 +6227,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6313,8 +6313,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6326,8 +6326,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6370,8 +6370,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6389,8 +6389,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6475,8 +6475,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6488,8 +6488,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6532,8 +6532,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 @@ -6551,8 +6551,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 @@ -6637,8 +6637,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm ; @@ -6650,8 +6650,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { @@ -6953,8 +6953,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -6969,8 +6969,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -7058,8 +7058,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 @@ -7072,8 +7072,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 @@ -7109,8 +7109,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -7125,8 +7125,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -7214,8 +7214,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 @@ -7228,8 +7228,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 @@ -7844,8 +7844,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw: @@ -7859,8 +7859,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_atomicrmw: @@ -7935,8 +7935,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acquire_atomicrmw: @@ -7949,8 +7949,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -8101,8 +8101,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw: @@ -8116,8 +8116,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_atomicrmw: @@ -8196,8 +8196,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw: @@ -8210,8 +8210,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -8244,8 +8244,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw: @@ -8259,8 +8259,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_atomicrmw: @@ -8339,8 +8339,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw: @@ -8353,8 +8353,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in) { entry: @@ -8389,8 +8389,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm @@ -8406,8 +8406,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm @@ -8491,8 +8491,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -8507,8 +8507,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -8546,8 +8546,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm @@ -8563,8 +8563,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm @@ -8652,8 +8652,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -8668,8 +8668,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -8707,8 +8707,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2 ; GFX10-WGP-NEXT: s_endpgm @@ -8724,8 +8724,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2 ; GFX10-CU-NEXT: s_endpgm @@ -8813,8 +8813,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -8829,8 +8829,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -8982,8 +8982,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg: @@ -8998,8 +8998,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg: @@ -9070,8 +9070,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg: @@ -9082,8 +9082,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9237,8 +9237,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg: @@ -9253,8 +9253,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg: @@ -9329,8 +9329,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg: @@ -9341,8 +9341,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9379,8 +9379,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg: @@ -9395,8 +9395,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg: @@ -9471,8 +9471,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg: @@ -9483,8 +9483,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9521,8 +9521,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg: @@ -9537,8 +9537,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg: @@ -9609,8 +9609,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg: @@ -9621,8 +9621,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9659,8 +9659,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg: @@ -9675,8 +9675,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_cmpxchg: @@ -9747,8 +9747,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg: @@ -9759,8 +9759,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9797,8 +9797,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg: @@ -9813,8 +9813,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_cmpxchg: @@ -9889,8 +9889,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg: @@ -9901,8 +9901,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -9939,8 +9939,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg: @@ -9955,8 +9955,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg: @@ -10031,8 +10031,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg: @@ -10043,8 +10043,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10081,8 +10081,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg: @@ -10097,8 +10097,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg: @@ -10173,8 +10173,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg: @@ -10185,8 +10185,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10223,8 +10223,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg: @@ -10239,8 +10239,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg: @@ -10315,8 +10315,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg: @@ -10327,8 +10327,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10365,8 +10365,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg: @@ -10381,8 +10381,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg: @@ -10457,8 +10457,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg: @@ -10469,8 +10469,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10507,8 +10507,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg: @@ -10523,8 +10523,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_cmpxchg: @@ -10599,8 +10599,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg: @@ -10611,8 +10611,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10649,8 +10649,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg: @@ -10665,8 +10665,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg: @@ -10741,8 +10741,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg: @@ -10753,8 +10753,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -10791,8 +10791,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg: @@ -10807,8 +10807,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg: @@ -10883,8 +10883,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg: @@ -10895,8 +10895,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr %out, i32 %in, i32 %old) { entry: @@ -11080,8 +11080,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11100,8 +11100,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11186,8 +11186,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11200,8 +11200,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11393,8 +11393,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11413,8 +11413,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11503,8 +11503,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11517,8 +11517,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11563,8 +11563,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11583,8 +11583,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11673,8 +11673,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11687,8 +11687,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11733,8 +11733,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11753,8 +11753,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -11839,8 +11839,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -11853,8 +11853,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -11899,8 +11899,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -11919,8 +11919,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12005,8 +12005,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12019,8 +12019,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12065,8 +12065,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12085,8 +12085,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12175,8 +12175,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12189,8 +12189,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12235,8 +12235,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12255,8 +12255,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12345,8 +12345,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12359,8 +12359,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12405,8 +12405,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12425,8 +12425,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12515,8 +12515,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12529,8 +12529,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12575,8 +12575,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12595,8 +12595,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12685,8 +12685,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12699,8 +12699,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12745,8 +12745,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12765,8 +12765,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -12855,8 +12855,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -12869,8 +12869,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -12915,8 +12915,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -12935,8 +12935,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -13025,8 +13025,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -13039,8 +13039,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -13085,8 +13085,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -13105,8 +13105,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -13195,8 +13195,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -13209,8 +13209,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm @@ -13255,8 +13255,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) @@ -13275,8 +13275,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) @@ -13365,8 +13365,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-WGP-NEXT: s_endpgm @@ -13379,8 +13379,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 ; GFX11-CU-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll index 0a7a07e36144..0449e4168296 100644 --- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll @@ -317,8 +317,8 @@ define amdgpu_kernel void @global_agent_acquire_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -329,8 +329,8 @@ define amdgpu_kernel void @global_agent_acquire_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -400,8 +400,8 @@ define amdgpu_kernel void @global_agent_acquire_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -414,8 +414,8 @@ define amdgpu_kernel void @global_agent_acquire_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -465,8 +465,8 @@ define amdgpu_kernel void @global_agent_seq_cst_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -477,8 +477,8 @@ define amdgpu_kernel void @global_agent_seq_cst_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -548,8 +548,8 @@ define amdgpu_kernel void @global_agent_seq_cst_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -562,8 +562,8 @@ define amdgpu_kernel void @global_agent_seq_cst_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1241,8 +1241,8 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acquire_atomicrmw: @@ -1255,8 +1255,8 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acquire_atomicrmw: @@ -1328,8 +1328,8 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acquire_atomicrmw: @@ -1341,8 +1341,8 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -1514,8 +1514,8 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acq_rel_atomicrmw: @@ -1528,8 +1528,8 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_atomicrmw: @@ -1603,8 +1603,8 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acq_rel_atomicrmw: @@ -1616,8 +1616,8 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -1662,8 +1662,8 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_seq_cst_atomicrmw: @@ -1676,8 +1676,8 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_atomicrmw: @@ -1751,8 +1751,8 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_seq_cst_atomicrmw: @@ -1764,8 +1764,8 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -1812,8 +1812,8 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -1827,8 +1827,8 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -1906,8 +1906,8 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1922,8 +1922,8 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1974,8 +1974,8 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -1989,8 +1989,8 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -2070,8 +2070,8 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2086,8 +2086,8 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2138,8 +2138,8 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -2153,8 +2153,8 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -2234,8 +2234,8 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2250,8 +2250,8 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2427,8 +2427,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acquire_monotonic_cmpxchg: @@ -2440,8 +2440,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acquire_monotonic_cmpxchg: @@ -2510,8 +2510,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acquire_monotonic_cmpxchg: @@ -2522,8 +2522,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -2698,8 +2698,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acq_rel_monotonic_cmpxchg: @@ -2711,8 +2711,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_monotonic_cmpxchg: @@ -2783,8 +2783,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acq_rel_monotonic_cmpxchg: @@ -2795,8 +2795,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -2845,8 +2845,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_seq_cst_monotonic_cmpxchg: @@ -2858,8 +2858,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_monotonic_cmpxchg: @@ -2930,8 +2930,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_seq_cst_monotonic_cmpxchg: @@ -2942,8 +2942,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -2992,8 +2992,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_monotonic_acquire_cmpxchg: @@ -3005,8 +3005,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_monotonic_acquire_cmpxchg: @@ -3075,8 +3075,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_monotonic_acquire_cmpxchg: @@ -3087,8 +3087,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3137,8 +3137,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acquire_acquire_cmpxchg: @@ -3150,8 +3150,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acquire_acquire_cmpxchg: @@ -3220,8 +3220,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acquire_acquire_cmpxchg: @@ -3232,8 +3232,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3282,8 +3282,8 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_release_acquire_cmpxchg: @@ -3295,8 +3295,8 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_release_acquire_cmpxchg: @@ -3367,8 +3367,8 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_release_acquire_cmpxchg: @@ -3379,8 +3379,8 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3429,8 +3429,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acq_rel_acquire_cmpxchg: @@ -3442,8 +3442,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_acquire_cmpxchg: @@ -3514,8 +3514,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acq_rel_acquire_cmpxchg: @@ -3526,8 +3526,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3576,8 +3576,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_seq_cst_acquire_cmpxchg: @@ -3589,8 +3589,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_acquire_cmpxchg: @@ -3661,8 +3661,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_seq_cst_acquire_cmpxchg: @@ -3673,8 +3673,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3723,8 +3723,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_monotonic_seq_cst_cmpxchg: @@ -3736,8 +3736,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_monotonic_seq_cst_cmpxchg: @@ -3808,8 +3808,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_monotonic_seq_cst_cmpxchg: @@ -3820,8 +3820,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3870,8 +3870,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acquire_seq_cst_cmpxchg: @@ -3883,8 +3883,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acquire_seq_cst_cmpxchg: @@ -3955,8 +3955,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acquire_seq_cst_cmpxchg: @@ -3967,8 +3967,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -4017,8 +4017,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_release_seq_cst_cmpxchg: @@ -4030,8 +4030,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_release_seq_cst_cmpxchg: @@ -4102,8 +4102,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_release_seq_cst_cmpxchg: @@ -4114,8 +4114,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -4164,8 +4164,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_acq_rel_seq_cst_cmpxchg: @@ -4177,8 +4177,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_seq_cst_cmpxchg: @@ -4249,8 +4249,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_acq_rel_seq_cst_cmpxchg: @@ -4261,8 +4261,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -4311,8 +4311,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_seq_cst_seq_cst_cmpxchg: @@ -4324,8 +4324,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_seq_cst_cmpxchg: @@ -4396,8 +4396,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_seq_cst_seq_cst_cmpxchg: @@ -4408,8 +4408,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -4612,8 +4612,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4626,8 +4626,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -4702,8 +4702,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4717,8 +4717,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4928,8 +4928,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4942,8 +4942,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5020,8 +5020,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5035,8 +5035,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5094,8 +5094,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5108,8 +5108,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5186,8 +5186,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5201,8 +5201,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5260,8 +5260,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5274,8 +5274,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5350,8 +5350,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5365,8 +5365,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5424,8 +5424,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5438,8 +5438,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5514,8 +5514,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5529,8 +5529,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5588,8 +5588,8 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5602,8 +5602,8 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5680,8 +5680,8 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5695,8 +5695,8 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5754,8 +5754,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5768,8 +5768,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5846,8 +5846,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5861,8 +5861,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5920,8 +5920,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5934,8 +5934,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6012,8 +6012,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6027,8 +6027,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6086,8 +6086,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6100,8 +6100,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6178,8 +6178,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6193,8 +6193,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6252,8 +6252,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6266,8 +6266,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6344,8 +6344,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6359,8 +6359,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6418,8 +6418,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6432,8 +6432,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6510,8 +6510,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6525,8 +6525,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6584,8 +6584,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6598,8 +6598,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6676,8 +6676,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6691,8 +6691,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6750,8 +6750,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6764,8 +6764,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6842,8 +6842,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6857,8 +6857,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -7178,8 +7178,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -7190,8 +7190,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -7261,8 +7261,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -7275,8 +7275,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -7326,8 +7326,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -7338,8 +7338,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -7409,8 +7409,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -7423,8 +7423,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8102,8 +8102,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acquire_atomicrmw: @@ -8116,8 +8116,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_atomicrmw: @@ -8189,8 +8189,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acquire_atomicrmw: @@ -8202,8 +8202,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -8375,8 +8375,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acq_rel_atomicrmw: @@ -8389,8 +8389,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_atomicrmw: @@ -8464,8 +8464,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acq_rel_atomicrmw: @@ -8477,8 +8477,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -8523,8 +8523,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_seq_cst_atomicrmw: @@ -8537,8 +8537,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_atomicrmw: @@ -8612,8 +8612,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_seq_cst_atomicrmw: @@ -8625,8 +8625,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -8673,8 +8673,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -8688,8 +8688,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -8767,8 +8767,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8783,8 +8783,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8835,8 +8835,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -8850,8 +8850,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -8931,8 +8931,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8947,8 +8947,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8999,8 +8999,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -9014,8 +9014,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -9095,8 +9095,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -9111,8 +9111,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -9288,8 +9288,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg: @@ -9301,8 +9301,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg: @@ -9371,8 +9371,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg: @@ -9383,8 +9383,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9559,8 +9559,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg: @@ -9572,8 +9572,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg: @@ -9644,8 +9644,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg: @@ -9656,8 +9656,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9706,8 +9706,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg: @@ -9719,8 +9719,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg: @@ -9791,8 +9791,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg: @@ -9803,8 +9803,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9853,8 +9853,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg: @@ -9866,8 +9866,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg: @@ -9936,8 +9936,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg: @@ -9948,8 +9948,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9998,8 +9998,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acquire_acquire_cmpxchg: @@ -10011,8 +10011,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_acquire_cmpxchg: @@ -10081,8 +10081,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acquire_acquire_cmpxchg: @@ -10093,8 +10093,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10143,8 +10143,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_release_acquire_cmpxchg: @@ -10156,8 +10156,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_acquire_cmpxchg: @@ -10228,8 +10228,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_release_acquire_cmpxchg: @@ -10240,8 +10240,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10290,8 +10290,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg: @@ -10303,8 +10303,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg: @@ -10375,8 +10375,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg: @@ -10387,8 +10387,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10437,8 +10437,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg: @@ -10450,8 +10450,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg: @@ -10522,8 +10522,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg: @@ -10534,8 +10534,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10584,8 +10584,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg: @@ -10597,8 +10597,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg: @@ -10669,8 +10669,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg: @@ -10681,8 +10681,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10731,8 +10731,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg: @@ -10744,8 +10744,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg: @@ -10816,8 +10816,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg: @@ -10828,8 +10828,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10878,8 +10878,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_release_seq_cst_cmpxchg: @@ -10891,8 +10891,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_seq_cst_cmpxchg: @@ -10963,8 +10963,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_release_seq_cst_cmpxchg: @@ -10975,8 +10975,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -11025,8 +11025,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg: @@ -11038,8 +11038,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg: @@ -11110,8 +11110,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg: @@ -11122,8 +11122,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -11172,8 +11172,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg: @@ -11185,8 +11185,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg: @@ -11257,8 +11257,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg: @@ -11269,8 +11269,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -11473,8 +11473,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11487,8 +11487,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11563,8 +11563,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11578,8 +11578,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11637,8 +11637,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11651,8 +11651,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11729,8 +11729,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11744,8 +11744,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11803,8 +11803,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11817,8 +11817,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11895,8 +11895,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11910,8 +11910,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11969,8 +11969,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11983,8 +11983,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12059,8 +12059,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12074,8 +12074,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12133,8 +12133,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12147,8 +12147,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12223,8 +12223,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12238,8 +12238,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12297,8 +12297,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12311,8 +12311,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12389,8 +12389,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12404,8 +12404,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12463,8 +12463,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12477,8 +12477,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12555,8 +12555,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12570,8 +12570,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12629,8 +12629,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12643,8 +12643,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12721,8 +12721,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12736,8 +12736,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12795,8 +12795,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12809,8 +12809,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12887,8 +12887,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12902,8 +12902,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12961,8 +12961,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12975,8 +12975,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -13053,8 +13053,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13068,8 +13068,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13127,8 +13127,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -13141,8 +13141,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -13219,8 +13219,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13234,8 +13234,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13293,8 +13293,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -13307,8 +13307,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -13385,8 +13385,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13400,8 +13400,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13459,8 +13459,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -13473,8 +13473,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -13551,8 +13551,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13566,8 +13566,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll index 97b2f0813e5e..f611003a7ba5 100644 --- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll @@ -317,8 +317,8 @@ define amdgpu_kernel void @global_system_acquire_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -329,8 +329,8 @@ define amdgpu_kernel void @global_system_acquire_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -402,8 +402,8 @@ define amdgpu_kernel void @global_system_acquire_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -416,8 +416,8 @@ define amdgpu_kernel void @global_system_acquire_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -467,8 +467,8 @@ define amdgpu_kernel void @global_system_seq_cst_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -479,8 +479,8 @@ define amdgpu_kernel void @global_system_seq_cst_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -552,8 +552,8 @@ define amdgpu_kernel void @global_system_seq_cst_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -566,8 +566,8 @@ define amdgpu_kernel void @global_system_seq_cst_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1249,8 +1249,8 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_acquire_atomicrmw: @@ -1263,8 +1263,8 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_acquire_atomicrmw: @@ -1338,8 +1338,8 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_acquire_atomicrmw: @@ -1351,8 +1351,8 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -1526,8 +1526,8 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_acq_rel_atomicrmw: @@ -1540,8 +1540,8 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_atomicrmw: @@ -1619,8 +1619,8 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_acq_rel_atomicrmw: @@ -1632,8 +1632,8 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -1678,8 +1678,8 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_seq_cst_atomicrmw: @@ -1692,8 +1692,8 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_atomicrmw: @@ -1771,8 +1771,8 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_seq_cst_atomicrmw: @@ -1784,8 +1784,8 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -1832,8 +1832,8 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -1847,8 +1847,8 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -1928,8 +1928,8 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1944,8 +1944,8 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1996,8 +1996,8 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -2011,8 +2011,8 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -2096,8 +2096,8 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2112,8 +2112,8 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2164,8 +2164,8 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -2179,8 +2179,8 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -2264,8 +2264,8 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2280,8 +2280,8 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2457,8 +2457,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_acquire_monotonic_cmpxchg: @@ -2470,8 +2470,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_acquire_monotonic_cmpxchg: @@ -2542,8 +2542,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_acquire_monotonic_cmpxchg: @@ -2554,8 +2554,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -2732,8 +2732,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_acq_rel_monotonic_cmpxchg: @@ -2745,8 +2745,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_monotonic_cmpxchg: @@ -2821,8 +2821,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_acq_rel_monotonic_cmpxchg: @@ -2833,8 +2833,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -2883,8 +2883,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_seq_cst_monotonic_cmpxchg: @@ -2896,8 +2896,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_monotonic_cmpxchg: @@ -2972,8 +2972,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_seq_cst_monotonic_cmpxchg: @@ -2984,8 +2984,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3034,8 +3034,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_monotonic_acquire_cmpxchg: @@ -3047,8 +3047,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_monotonic_acquire_cmpxchg: @@ -3119,8 +3119,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_monotonic_acquire_cmpxchg: @@ -3131,8 +3131,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3181,8 +3181,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_acquire_acquire_cmpxchg: @@ -3194,8 +3194,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_acquire_acquire_cmpxchg: @@ -3266,8 +3266,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_acquire_acquire_cmpxchg: @@ -3278,8 +3278,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3328,8 +3328,8 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_release_acquire_cmpxchg: @@ -3341,8 +3341,8 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_release_acquire_cmpxchg: @@ -3417,8 +3417,8 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_release_acquire_cmpxchg: @@ -3429,8 +3429,8 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3479,8 +3479,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_acq_rel_acquire_cmpxchg: @@ -3492,8 +3492,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_acquire_cmpxchg: @@ -3568,8 +3568,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_acq_rel_acquire_cmpxchg: @@ -3580,8 +3580,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3630,8 +3630,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_seq_cst_acquire_cmpxchg: @@ -3643,8 +3643,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_acquire_cmpxchg: @@ -3719,8 +3719,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_seq_cst_acquire_cmpxchg: @@ -3731,8 +3731,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -3781,8 +3781,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_seq_cst_seq_cst_cmpxchg: @@ -3794,8 +3794,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_seq_cst_cmpxchg: @@ -3870,8 +3870,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_seq_cst_seq_cst_cmpxchg: @@ -3882,8 +3882,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -4086,8 +4086,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4100,8 +4100,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -4178,8 +4178,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4193,8 +4193,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4252,8 +4252,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4266,8 +4266,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -4348,8 +4348,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4363,8 +4363,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4422,8 +4422,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4436,8 +4436,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -4518,8 +4518,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4533,8 +4533,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4592,8 +4592,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4606,8 +4606,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -4684,8 +4684,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4699,8 +4699,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4758,8 +4758,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4772,8 +4772,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -4850,8 +4850,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4865,8 +4865,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4924,8 +4924,8 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -4938,8 +4938,8 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5020,8 +5020,8 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5035,8 +5035,8 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5094,8 +5094,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5108,8 +5108,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5190,8 +5190,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5205,8 +5205,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5264,8 +5264,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5278,8 +5278,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5360,8 +5360,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5375,8 +5375,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5434,8 +5434,8 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5448,8 +5448,8 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5530,8 +5530,8 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5545,8 +5545,8 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5604,8 +5604,8 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5618,8 +5618,8 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5700,8 +5700,8 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5715,8 +5715,8 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5774,8 +5774,8 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5788,8 +5788,8 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -5870,8 +5870,8 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5885,8 +5885,8 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -5944,8 +5944,8 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -5958,8 +5958,8 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6040,8 +6040,8 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6055,8 +6055,8 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6114,8 +6114,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6128,8 +6128,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -6210,8 +6210,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6225,8 +6225,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6546,8 +6546,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6558,8 +6558,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -6631,8 +6631,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6645,8 +6645,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6696,8 +6696,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load( ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-WGP-NEXT: s_endpgm ; @@ -6708,8 +6708,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load( ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-CU-NEXT: global_load_dword v1, v0, s[0:1] glc dlc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[2:3] ; GFX10-CU-NEXT: s_endpgm ; @@ -6781,8 +6781,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load( ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -6795,8 +6795,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load( ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[2:3] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -7478,8 +7478,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acquire_atomicrmw: @@ -7492,8 +7492,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_atomicrmw: @@ -7567,8 +7567,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acquire_atomicrmw: @@ -7580,8 +7580,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -7755,8 +7755,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acq_rel_atomicrmw: @@ -7769,8 +7769,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_atomicrmw: @@ -7848,8 +7848,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acq_rel_atomicrmw: @@ -7861,8 +7861,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -7907,8 +7907,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_seq_cst_atomicrmw: @@ -7921,8 +7921,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[0:1] ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_atomicrmw: @@ -8000,8 +8000,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_seq_cst_atomicrmw: @@ -8013,8 +8013,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in) { entry: @@ -8061,8 +8061,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -8076,8 +8076,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -8157,8 +8157,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8173,8 +8173,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8225,8 +8225,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -8240,8 +8240,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -8325,8 +8325,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8341,8 +8341,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8393,8 +8393,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -8408,8 +8408,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s2 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[0:1] glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v0, v1, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -8493,8 +8493,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8509,8 +8509,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8686,8 +8686,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acquire_monotonic_cmpxchg: @@ -8699,8 +8699,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_monotonic_cmpxchg: @@ -8771,8 +8771,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acquire_monotonic_cmpxchg: @@ -8783,8 +8783,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -8961,8 +8961,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg: @@ -8974,8 +8974,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg: @@ -9050,8 +9050,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg: @@ -9062,8 +9062,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9112,8 +9112,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg: @@ -9125,8 +9125,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg: @@ -9201,8 +9201,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg: @@ -9213,8 +9213,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9263,8 +9263,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_monotonic_acquire_cmpxchg: @@ -9276,8 +9276,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_acquire_cmpxchg: @@ -9348,8 +9348,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_monotonic_acquire_cmpxchg: @@ -9360,8 +9360,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9410,8 +9410,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acquire_acquire_cmpxchg: @@ -9423,8 +9423,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_acquire_cmpxchg: @@ -9495,8 +9495,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acquire_acquire_cmpxchg: @@ -9507,8 +9507,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9557,8 +9557,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_release_acquire_cmpxchg: @@ -9570,8 +9570,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_acquire_cmpxchg: @@ -9646,8 +9646,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_release_acquire_cmpxchg: @@ -9658,8 +9658,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9708,8 +9708,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg: @@ -9721,8 +9721,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg: @@ -9797,8 +9797,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg: @@ -9809,8 +9809,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -9859,8 +9859,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg: @@ -9872,8 +9872,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg: @@ -9948,8 +9948,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg: @@ -9960,8 +9960,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10010,8 +10010,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg: @@ -10023,8 +10023,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg: @@ -10099,8 +10099,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg: @@ -10111,8 +10111,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10161,8 +10161,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg: @@ -10174,8 +10174,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg: @@ -10250,8 +10250,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg: @@ -10262,8 +10262,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10312,8 +10312,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_release_seq_cst_cmpxchg: @@ -10325,8 +10325,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_seq_cst_cmpxchg: @@ -10401,8 +10401,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_release_seq_cst_cmpxchg: @@ -10413,8 +10413,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10463,8 +10463,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg: @@ -10476,8 +10476,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg: @@ -10552,8 +10552,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg: @@ -10564,8 +10564,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10614,8 +10614,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: s_endpgm ; ; GFX10-CU-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg: @@ -10627,8 +10627,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: s_endpgm ; ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg: @@ -10703,8 +10703,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: s_endpgm ; ; GFX11-CU-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg: @@ -10715,8 +10715,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: s_endpgm ptr addrspace(1) %out, i32 %in, i32 %old) { entry: @@ -10919,8 +10919,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -10933,8 +10933,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11011,8 +11011,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11026,8 +11026,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11239,8 +11239,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11253,8 +11253,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11335,8 +11335,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11350,8 +11350,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11409,8 +11409,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11423,8 +11423,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11505,8 +11505,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11520,8 +11520,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11579,8 +11579,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11593,8 +11593,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11671,8 +11671,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11686,8 +11686,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11745,8 +11745,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11759,8 +11759,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -11837,8 +11837,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11852,8 +11852,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -11911,8 +11911,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -11925,8 +11925,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12007,8 +12007,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12022,8 +12022,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12081,8 +12081,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12095,8 +12095,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12177,8 +12177,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12192,8 +12192,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12251,8 +12251,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12265,8 +12265,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12347,8 +12347,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12362,8 +12362,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12421,8 +12421,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12435,8 +12435,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12517,8 +12517,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12532,8 +12532,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12591,8 +12591,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12605,8 +12605,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12687,8 +12687,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12702,8 +12702,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12761,8 +12761,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12775,8 +12775,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -12857,8 +12857,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12872,8 +12872,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -12931,8 +12931,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -12945,8 +12945,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -13027,8 +13027,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13042,8 +13042,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13101,8 +13101,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: buffer_gl1_inv +; GFX10-WGP-NEXT: buffer_gl0_inv ; GFX10-WGP-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-WGP-NEXT: s_endpgm ; @@ -13115,8 +13115,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) -; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: buffer_gl1_inv +; GFX10-CU-NEXT: buffer_gl0_inv ; GFX10-CU-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-CU-NEXT: s_endpgm ; @@ -13197,8 +13197,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) -; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: buffer_gl1_inv +; GFX11-WGP-NEXT: buffer_gl0_inv ; GFX11-WGP-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-WGP-NEXT: s_nop 0 ; GFX11-WGP-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -13212,8 +13212,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg( ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) -; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: buffer_gl1_inv +; GFX11-CU-NEXT: buffer_gl0_inv ; GFX11-CU-NEXT: global_store_b32 v2, v0, s[0:1] ; GFX11-CU-NEXT: s_nop 0 ; GFX11-CU-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) -- GitLab From e892f323ea234a997888b0fa44356b0678eda5f0 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Tue, 13 Feb 2024 09:17:09 +0100 Subject: [PATCH 023/284] [mlir][nvgpu] Allow TMA's last dim to be non-128B without swizzling (#81499) Allow TMA's last dimension to be non-128B when swizzling mode is not set. Test `tma_load_64x8_8x128_noswizzle.mlir` is failing due to the verifier. This PR will fix that --- mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp index 4b6327479a21..26f831f10a4e 100644 --- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp +++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp @@ -362,7 +362,8 @@ std::optional verifyTmaDescriptorWithMemref( << kMaxTMADimension << " but it is " << dim; } } - if (descMemref.getRank() > 1) { + if (descMemref.getRank() > 1 && + descType.getSwizzle() != TensorMapSwizzleKind::SWIZZLE_NONE) { unsigned lastDimensionByte = descMemref.getElementTypeBitWidth() * descMemref.getShape().back() / 8; if (lastDimensionByte != kMaxTMALastdimByte) -- GitLab From 346e7c7f6881afaade5a71ad97475d70639dadcf Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Tue, 13 Feb 2024 12:17:46 +0400 Subject: [PATCH 024/284] [clang] Add some CodeGen tests for CWG 2xx issues (#80823) This patch covers CWG issues [201](https://cplusplus.github.io/CWG/issues/201.html), [210](https://cplusplus.github.io/CWG/issues/210.html), [292](https://cplusplus.github.io/CWG/issues/292.html). [CWG208](https://cplusplus.github.io/CWG/issues/208.html) is not covered, as it actually requires a libcxxabi test. Resolution of CWG292 has been superseded by [P0145R3](https://wg21.link/p0145r3) "Refining Expression Evaluation Order for Idiomatic C++" (see changes to paragraph 5.3.4/18). --- clang/test/CXX/drs/dr201.cpp | 42 ++++++++++++++++++++++++++++++++++++ clang/test/CXX/drs/dr210.cpp | 41 +++++++++++++++++++++++++++++++++++ clang/test/CXX/drs/dr292.cpp | 30 ++++++++++++++++++++++++++ clang/test/CXX/drs/dr2xx.cpp | 6 +++--- clang/www/cxx_dr_status.html | 6 +++--- 5 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 clang/test/CXX/drs/dr201.cpp create mode 100644 clang/test/CXX/drs/dr210.cpp create mode 100644 clang/test/CXX/drs/dr292.cpp diff --git a/clang/test/CXX/drs/dr201.cpp b/clang/test/CXX/drs/dr201.cpp new file mode 100644 index 000000000000..7e864981e13b --- /dev/null +++ b/clang/test/CXX/drs/dr201.cpp @@ -0,0 +1,42 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +#if __cplusplus == 199711L +#define NOTHROW throw() +#else +#define NOTHROW noexcept(true) +#endif + +namespace dr201 { // dr201: 2.8 + +extern void full_expr_fence() NOTHROW; + +struct A { + ~A() NOTHROW {} +}; + +struct B { + B(A) NOTHROW {} + ~B() NOTHROW {} +}; + +void foo() { + full_expr_fence(); + B b = A(); + full_expr_fence(); +} + +// CHECK-LABEL: define {{.*}} void @dr201::foo() +// CHECK: call void @dr201::full_expr_fence() +// CHECK: call void @dr201::B::B(dr201::A) +// CHECK: call void @dr201::A::~A() +// CHECK: call void @dr201::full_expr_fence() +// CHECK: call void @dr201::B::~B() +// CHECK-LABEL: } + +} // namespace dr201 diff --git a/clang/test/CXX/drs/dr210.cpp b/clang/test/CXX/drs/dr210.cpp new file mode 100644 index 000000000000..156ee81093b4 --- /dev/null +++ b/clang/test/CXX/drs/dr210.cpp @@ -0,0 +1,41 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +#if __cplusplus == 199711L +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wvariadic-macros" +#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +#pragma clang diagnostic pop +#endif + +namespace dr210 { // dr210: 2.7 +struct B { + long i; + B(); + virtual ~B(); +}; + +static_assert(sizeof(B) == 16, ""); + +struct D : B { + long j; + D(); +}; + +static_assert(sizeof(D) == 24, ""); + +void toss(const B* b) { + throw *b; +} + +// CHECK-LABEL: define {{.*}} void @dr210::toss(dr210::B const*) +// CHECK: %[[EXCEPTION:.*]] = call ptr @__cxa_allocate_exception(i64 16) +// CHECK: call void @__cxa_throw(ptr %[[EXCEPTION]], ptr @typeinfo for dr210::B, ptr @dr210::B::~B()) +// CHECK-LABEL: } + +} // namespace dr210 diff --git a/clang/test/CXX/drs/dr292.cpp b/clang/test/CXX/drs/dr292.cpp new file mode 100644 index 000000000000..19caeef291fa --- /dev/null +++ b/clang/test/CXX/drs/dr292.cpp @@ -0,0 +1,30 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +namespace dr292 { // dr292: 2.9 + +extern int g(); + +struct A { + A(int) throw() {} +}; + +void f() { + new A(g()); +} + +// CHECK-LABEL: define {{.*}} void @dr292::f()() +// CHECK: %[[CALL:.+]] = call {{.*}} @operator new(unsigned long)({{.*}}) +// CHECK: invoke {{.*}} i32 @dr292::g()() +// CHECK-NEXT: to {{.*}} unwind label %lpad +// CHECK-LABEL: lpad: +// CHECK: call void @operator delete(void*)(ptr {{.*}} %[[CALL]]) +// CHECK-LABEL: eh.resume: +// CHECK-LABEL: } + +} // namespace dr292 diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/dr2xx.cpp index 1a3ac532f93b..cbb8734e10c6 100644 --- a/clang/test/CXX/drs/dr2xx.cpp +++ b/clang/test/CXX/drs/dr2xx.cpp @@ -26,7 +26,7 @@ namespace dr200 { // dr200: dup 214 } } -// dr201 FIXME: write codegen test +// dr201 is in dr201.cpp namespace dr202 { // dr202: 3.1 template T f(); @@ -76,7 +76,7 @@ namespace dr209 { // dr209: 3.2 }; } -// dr210 FIXME: write codegen test +// dr210 is in dr210.cpp namespace dr211 { // dr211: yes struct A { @@ -1188,7 +1188,7 @@ namespace dr289 { // dr289: yes // dr290: na // dr291: dup 391 -// dr292 FIXME: write a codegen test +// dr292 is in dr292.cpp namespace dr294 { // dr294: no void f() throw(int); diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index 8f4ae23505f0..1e527e1d1736 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -1244,7 +1244,7 @@ 201 CD1 Order of destruction of temporaries in initializers - Unknown + Clang 2.8 202 @@ -1299,7 +1299,7 @@ accessible? 210 TC1 What is the type matched by an exception handler? - Unknown + Clang 2.7 211 @@ -1792,7 +1792,7 @@ of class templates 292 CD3 Deallocation on exception in new before arguments evaluated - Unknown + Clang 2.9 293 -- GitLab From d57515bd107bc76df5a042ffee2b7dc6125ffef1 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 13 Feb 2024 08:21:35 +0000 Subject: [PATCH 025/284] [LLT] Add and use isPointerVector and isPointerOrPointerVector. NFC. (#81283) --- .../llvm/CodeGen/GlobalISel/LegalizerInfo.h | 2 +- llvm/include/llvm/CodeGenTypes/LowLevelType.h | 14 ++++++++------ llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 5 ++--- llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 2 +- llvm/lib/CodeGen/MachineVerifier.cpp | 14 ++++++-------- .../AArch64/GISel/AArch64InstructionSelector.cpp | 2 +- .../Target/AArch64/GISel/AArch64LegalizerInfo.cpp | 11 ++++------- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 5 ++--- llvm/unittests/CodeGen/LowLevelTypeTest.cpp | 3 +++ 9 files changed, 28 insertions(+), 30 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 5fd80e5f199c..637c2c71b024 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -1084,7 +1084,7 @@ public: }, [=](const LegalityQuery &Query) { LLT T = Query.Types[LargeTypeIdx]; - if (T.isVector() && T.getElementType().isPointer()) + if (T.isPointerVector()) T = T.changeElementType(LLT::scalar(T.getScalarSizeInBits())); return std::make_pair(TypeIdx, T); }); diff --git a/llvm/include/llvm/CodeGenTypes/LowLevelType.h b/llvm/include/llvm/CodeGenTypes/LowLevelType.h index cc33152e06f4..5a16cffb80b3 100644 --- a/llvm/include/llvm/CodeGenTypes/LowLevelType.h +++ b/llvm/include/llvm/CodeGenTypes/LowLevelType.h @@ -134,15 +134,17 @@ public: explicit LLT(MVT VT); - constexpr bool isValid() const { return IsScalar || RawData != 0; } + constexpr bool isValid() const { return IsScalar || IsPointer || IsVector; } constexpr bool isScalar() const { return IsScalar; } - constexpr bool isPointer() const { - return isValid() && IsPointer && !IsVector; - } + constexpr bool isPointer() const { return IsPointer && !IsVector; } + + constexpr bool isPointerVector() const { return IsPointer && IsVector; } + + constexpr bool isPointerOrPointerVector() const { return IsPointer; } - constexpr bool isVector() const { return isValid() && IsVector; } + constexpr bool isVector() const { return IsVector; } /// Returns the number of elements in a vector LLT. Must only be called on /// vector types. @@ -209,7 +211,7 @@ public: /// but the new element size. Otherwise, return the new element type. Invalid /// for pointer types. For pointer types, use changeElementType. constexpr LLT changeElementSize(unsigned NewEltSize) const { - assert(!getScalarType().isPointer() && + assert(!isPointerOrPointerVector() && "invalid to directly change element size for pointers"); return isVector() ? LLT::vector(getElementCount(), NewEltSize) : LLT::scalar(NewEltSize); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 464ff0864d14..e39fdae1ccbe 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -1716,8 +1716,7 @@ Register LegalizerHelper::coerceToScalar(Register Val) { Register NewVal = Val; assert(Ty.isVector()); - LLT EltTy = Ty.getElementType(); - if (EltTy.isPointer()) + if (Ty.isPointerVector()) NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0); return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0); } @@ -7964,7 +7963,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) { auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] = MI.getFirst4RegLLTs(); - bool IsEltPtr = DstTy.getScalarType().isPointer(); + bool IsEltPtr = DstTy.isPointerOrPointerVector(); if (IsEltPtr) { LLT ScalarPtrTy = LLT::scalar(DstTy.getScalarSizeInBits()); LLT NewTy = DstTy.changeElementType(ScalarPtrTy); diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index a5827c26c04f..d58b62871817 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -199,7 +199,7 @@ void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0, MachineInstrBuilder MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional Flags) { - assert(Res.getLLTTy(*getMRI()).getScalarType().isPointer() && + assert(Res.getLLTTy(*getMRI()).isPointerOrPointerVector() && Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch"); assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type"); diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index c65e91741533..2632b5b9feac 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1288,10 +1288,10 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid()) break; - if (!PtrTy.getScalarType().isPointer()) + if (!PtrTy.isPointerOrPointerVector()) report("gep first operand must be a pointer", MI); - if (OffsetTy.getScalarType().isPointer()) + if (OffsetTy.isPointerOrPointerVector()) report("gep offset operand must not be a pointer", MI); // TODO: Is the offset allowed to be a scalar with a vector? @@ -1304,7 +1304,7 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid()) break; - if (!DstTy.getScalarType().isPointer()) + if (!DstTy.isPointerOrPointerVector()) report("ptrmask result type must be a pointer", MI); if (!MaskTy.getScalarType().isScalar()) @@ -1330,15 +1330,13 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { if (!DstTy.isValid() || !SrcTy.isValid()) break; - LLT DstElTy = DstTy.getScalarType(); - LLT SrcElTy = SrcTy.getScalarType(); - if (DstElTy.isPointer() || SrcElTy.isPointer()) + if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector()) report("Generic extend/truncate can not operate on pointers", MI); verifyVectorElementMatch(DstTy, SrcTy, MI); - unsigned DstSize = DstElTy.getSizeInBits(); - unsigned SrcSize = SrcElTy.getSizeInBits(); + unsigned DstSize = DstTy.getScalarSizeInBits(); + unsigned SrcSize = SrcTy.getScalarSizeInBits(); switch (MI->getOpcode()) { default: if (DstSize <= SrcSize) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 9d51a7f7616d..ac804857be01 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2091,7 +2091,7 @@ bool AArch64InstructionSelector::preISelLower(MachineInstr &I) { case AArch64::G_DUP: { // Convert the type from p0 to s64 to help selection. LLT DstTy = MRI.getType(I.getOperand(0).getReg()); - if (!DstTy.getElementType().isPointer()) + if (!DstTy.isPointerVector()) return false; auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg()); MRI.setType(I.getOperand(0).getReg(), diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index cbf5655706e6..ab25e2b9af8b 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -343,10 +343,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) auto IsPtrVecPred = [=](const LegalityQuery &Query) { const LLT &ValTy = Query.Types[0]; - if (!ValTy.isVector()) - return false; - const LLT EltTy = ValTy.getElementType(); - return EltTy.isPointer() && EltTy.getAddressSpace() == 0; + return ValTy.isPointerVector() && ValTy.getAddressSpace() == 0; }; getActionDefinitionsBuilder(G_LOAD) @@ -521,7 +518,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) [=](const LegalityQuery &Query) { const LLT &Ty = Query.Types[0]; const LLT &SrcTy = Query.Types[1]; - return Ty.isVector() && !SrcTy.getElementType().isPointer() && + return Ty.isVector() && !SrcTy.isPointerVector() && Ty.getElementType() != SrcTy.getElementType(); }, 0, 1) @@ -555,7 +552,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) [=](const LegalityQuery &Query) { const LLT &Ty = Query.Types[0]; const LLT &SrcTy = Query.Types[1]; - return Ty.isVector() && !SrcTy.getElementType().isPointer() && + return Ty.isVector() && !SrcTy.isPointerVector() && Ty.getElementType() != SrcTy.getElementType(); }, 0, 1) @@ -1649,7 +1646,7 @@ bool AArch64LegalizerInfo::legalizeLoadStore( return true; } - if (!ValTy.isVector() || !ValTy.getElementType().isPointer() || + if (!ValTy.isPointerVector() || ValTy.getElementType().getAddressSpace() != 0) { LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store"); return false; diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index def08cc89b41..f3716f96c44a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -498,11 +498,10 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) { if (!Ty.isVector()) return true; - LLT EltTy = Ty.getElementType(); - if (EltTy.isPointer()) + if (Ty.isPointerVector()) return true; - unsigned EltSize = EltTy.getSizeInBits(); + unsigned EltSize = Ty.getScalarSizeInBits(); return EltSize != 32 && EltSize != 64; } diff --git a/llvm/unittests/CodeGen/LowLevelTypeTest.cpp b/llvm/unittests/CodeGen/LowLevelTypeTest.cpp index d13cfeeffe56..cb34802a5de2 100644 --- a/llvm/unittests/CodeGen/LowLevelTypeTest.cpp +++ b/llvm/unittests/CodeGen/LowLevelTypeTest.cpp @@ -259,6 +259,7 @@ TEST(LowLevelTypeTest, Pointer) { // Test kind. ASSERT_TRUE(Ty.isValid()); ASSERT_TRUE(Ty.isPointer()); + ASSERT_TRUE(Ty.isPointerOrPointerVector()); ASSERT_FALSE(Ty.isScalar()); ASSERT_FALSE(Ty.isVector()); @@ -266,6 +267,8 @@ TEST(LowLevelTypeTest, Pointer) { ASSERT_TRUE(VTy.isValid()); ASSERT_TRUE(VTy.isVector()); ASSERT_TRUE(VTy.getElementType().isPointer()); + ASSERT_TRUE(VTy.isPointerVector()); + ASSERT_TRUE(VTy.isPointerOrPointerVector()); EXPECT_EQ(Ty, VTy.getElementType()); EXPECT_EQ(Ty.getSizeInBits(), VTy.getScalarSizeInBits()); -- GitLab From 0e16950e741aee7ffec865c81596e1411471475e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 13 Feb 2024 08:22:39 +0000 Subject: [PATCH 026/284] [X86] IceLakeServer - ZMM FADD/FMUL can only use Port0 Fix discrepancy from when this was forked from the SkylakeServer model This also fixes VRANGEPS/VRANGEPD instructions which typically match FADD characteristics Confirmed with Agner + uops.info Fixes #81504 --- llvm/lib/Target/X86/X86SchedIceLake.td | 8 +- .../X86/IceLakeServer/resources-avx512.s | 218 +++++++++--------- .../X86/IceLakeServer/resources-avx512dq.s | 98 ++++---- 3 files changed, 162 insertions(+), 162 deletions(-) diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index 2c660fad2ec7..3144327670c7 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -257,11 +257,11 @@ defm : X86WriteRes; defm : ICXWriteResPair; // Floating point add/sub. defm : ICXWriteResPair; defm : ICXWriteResPair; -defm : ICXWriteResPair; +defm : ICXWriteResPair; defm : ICXWriteResPair; // Floating point double add/sub. defm : ICXWriteResPair; defm : ICXWriteResPair; -defm : ICXWriteResPair; +defm : ICXWriteResPair; defm : ICXWriteResPair; // Floating point compare. defm : ICXWriteResPair; @@ -278,11 +278,11 @@ defm : ICXWriteResPair; // Floating point compa defm : ICXWriteResPair; // Floating point multiplication. defm : ICXWriteResPair; defm : ICXWriteResPair; -defm : ICXWriteResPair; +defm : ICXWriteResPair; defm : ICXWriteResPair; // Floating point double multiplication. defm : ICXWriteResPair; defm : ICXWriteResPair; -defm : ICXWriteResPair; +defm : ICXWriteResPair; defm : ICXWriteResPair; // 10-14 cycles. // Floating point division. defm : ICXWriteResPair; // 10-14 cycles. diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512.s index 6d33fdb3359b..5c12c520b04a 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512.s @@ -1084,24 +1084,24 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: 1 4 1.00 kshiftlw $2, %k1, %k2 # CHECK-NEXT: 1 4 1.00 kshiftrw $2, %k1, %k2 # CHECK-NEXT: 1 4 1.00 kunpckbw %k0, %k1, %k2 -# CHECK-NEXT: 1 4 0.50 vaddpd %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vaddpd (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vaddpd (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vaddpd %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vaddpd (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vaddpd %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vaddpd (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vaddps %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vaddps (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vaddps (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vaddps %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vaddps (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vaddps %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vaddps (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vaddpd %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vaddpd (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vaddpd (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vaddpd %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vaddpd (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vaddpd %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vaddpd (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vaddps %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vaddps (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vaddps (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vaddps %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vaddps (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vaddps %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vaddps (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: 1 3 1.00 valignd $1, %zmm16, %zmm17, %zmm19 # CHECK-NEXT: 2 10 1.00 * valignd $1, (%rax), %zmm17, %zmm19 # CHECK-NEXT: 2 10 1.00 * valignd $1, (%rax){1to16}, %zmm17, %zmm19 @@ -1510,24 +1510,24 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: 2 1 0.50 * vmovups %zmm16, (%rax) {%k1} # CHECK-NEXT: 1 1 0.50 vmovups %zmm16, %zmm19 {%k1} {z} # CHECK-NEXT: 2 8 0.50 * vmovups (%rax), %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vmulpd %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vmulpd (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vmulpd (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vmulpd %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vmulpd (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vmulpd %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vmulpd (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vmulps %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vmulps (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vmulps (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vmulps %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vmulps (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vmulps %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vmulps (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vmulpd %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vmulpd (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vmulpd (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vmulpd %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vmulpd (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vmulpd %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vmulpd (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vmulps %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vmulps (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vmulps (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vmulps %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vmulps (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vmulps %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vmulps (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: 1 1 1.00 vpabsd %zmm16, %zmm19 # CHECK-NEXT: 2 8 1.00 * vpabsd (%rax), %zmm19 # CHECK-NEXT: 2 8 1.00 * vpabsd (%rax){1to16}, %zmm19 @@ -1958,24 +1958,24 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: 1 4 1.00 vptestnmq %zmm0, %zmm1, %k2 {%k3} # CHECK-NEXT: 2 11 1.00 * vptestnmq (%rax), %zmm1, %k2 {%k3} # CHECK-NEXT: 2 11 1.00 * vptestnmq (%rax){1to8}, %zmm1, %k2 {%k3} -# CHECK-NEXT: 1 4 0.50 vsubpd %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vsubpd (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vsubpd (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vsubpd %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vsubpd (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vsubpd %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vsubpd (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vsubps %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vsubps (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vsubps (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vsubps %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vsubps (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vsubps %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vsubps (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vsubpd %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vsubpd (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vsubpd (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vsubpd %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vsubpd (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vsubpd %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vsubpd (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vsubps %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vsubps (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vsubps (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vsubps %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vsubps (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vsubps %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vsubps (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: 1 2 1.00 vucomiss %xmm16, %xmm17 # CHECK-NEXT: 2 7 1.00 * vucomiss (%rax), %xmm17 # CHECK-NEXT: 1 1 1.00 vunpckhpd %zmm16, %zmm17, %zmm19 @@ -2031,7 +2031,7 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - 612.00 371.17 99.67 327.50 327.50 8.00 612.17 2.00 8.00 8.00 8.00 +# CHECK-NEXT: - 612.00 398.17 99.67 327.50 327.50 8.00 585.17 2.00 8.00 8.00 8.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -2044,24 +2044,24 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: - - - - - - - 1.00 - - - - kshiftlw $2, %k1, %k2 # CHECK-NEXT: - - - - - - - 1.00 - - - - kshiftrw $2, %k1, %k2 # CHECK-NEXT: - - - - - - - 1.00 - - - - kunpckbw %k0, %k1, %k2 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vaddpd %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddpd (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddpd (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vaddpd %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddpd (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vaddpd %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddpd (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vaddps %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddps (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddps (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vaddps %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddps (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vaddps %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddps (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vaddpd %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddpd (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddpd (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vaddpd %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddpd (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vaddpd %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddpd (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vaddps %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddps (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddps (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vaddps %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddps (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vaddps %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddps (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vaddps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: - - - - - - - 1.00 - - - - valignd $1, %zmm16, %zmm17, %zmm19 # CHECK-NEXT: - - - - 0.50 0.50 - 1.00 - - - - valignd $1, (%rax), %zmm17, %zmm19 # CHECK-NEXT: - - - - 0.50 0.50 - 1.00 - - - - valignd $1, (%rax){1to16}, %zmm17, %zmm19 @@ -2470,24 +2470,24 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: - - - - - - 0.50 - - 0.50 0.50 0.50 vmovups %zmm16, (%rax) {%k1} # CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmovups %zmm16, %zmm19 {%k1} {z} # CHECK-NEXT: - - 0.33 0.33 0.50 0.50 - 0.33 - - - - vmovups (%rax), %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmulpd %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulpd (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulpd (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmulpd %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulpd (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmulpd %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulpd (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmulps %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulps (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulps (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmulps %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulps (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vmulps %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulps (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vmulpd %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulpd (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulpd (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vmulpd %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulpd (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vmulpd %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulpd (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vmulps %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulps (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulps (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vmulps %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulps (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vmulps %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulps (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vmulps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: - - 1.00 - - - - - - - - - vpabsd %zmm16, %zmm19 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vpabsd (%rax), %zmm19 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vpabsd (%rax){1to16}, %zmm19 @@ -2918,24 +2918,24 @@ vunpcklps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: - - - - - - - 1.00 - - - - vptestnmq %zmm0, %zmm1, %k2 {%k3} # CHECK-NEXT: - - - - 0.50 0.50 - 1.00 - - - - vptestnmq (%rax), %zmm1, %k2 {%k3} # CHECK-NEXT: - - - - 0.50 0.50 - 1.00 - - - - vptestnmq (%rax){1to8}, %zmm1, %k2 {%k3} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vsubpd %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubpd (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubpd (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vsubpd %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubpd (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vsubpd %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubpd (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vsubps %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubps (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubps (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vsubps %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubps (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vsubps %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubps (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vsubpd %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubpd (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubpd (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vsubpd %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubpd (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vsubpd %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubpd (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubpd (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vsubps %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubps (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubps (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vsubps %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubps (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vsubps %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubps (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vsubps (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: - - 1.00 - - - - - - - - - vucomiss %xmm16, %xmm17 # CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vucomiss (%rax), %xmm17 # CHECK-NEXT: - - - - - - - 1.00 - - - - vunpckhpd %zmm16, %zmm17, %zmm19 diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s index 4da4cebd6099..42041cd389fc 100644 --- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s +++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512dq.s @@ -753,30 +753,30 @@ vxorps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: 4 22 3.00 * vpmullq (%rax), %zmm17, %zmm19 {%k1} # CHECK-NEXT: 3 15 3.00 vpmullq %zmm16, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: 4 22 3.00 * vpmullq (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vrangepd $ab, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vrangepd $ab, (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vrangeps $ab, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vrangeps $ab, (%rax), %zmm17, %zmm19 -# CHECK-NEXT: 2 11 0.50 * vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 2 11 0.50 * vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 2 11 0.50 * vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: 1 4 0.50 vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: 1 4 0.50 vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: 1 4 0.50 vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vrangepd $ab, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vrangepd $ab, (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vrangeps $ab, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vrangeps $ab, (%rax), %zmm17, %zmm19 +# CHECK-NEXT: 2 11 1.00 * vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 2 11 1.00 * vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 2 11 1.00 * vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: 1 4 1.00 vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: 1 4 1.00 vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: 1 4 1.00 vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: 1 4 0.50 vrangesd $ab, %xmm16, %xmm17, %xmm19 # CHECK-NEXT: 2 10 0.50 * vrangesd $ab, (%rax), %xmm17, %xmm19 # CHECK-NEXT: 1 4 0.50 vrangesd $ab, %xmm16, %xmm17, %xmm19 {%k1} @@ -872,7 +872,7 @@ vxorps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] -# CHECK-NEXT: - - 197.25 73.25 102.00 102.00 5.50 209.25 0.25 5.50 5.50 5.50 +# CHECK-NEXT: - - 209.25 73.25 102.00 102.00 5.50 197.25 0.25 5.50 5.50 5.50 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions: @@ -1162,30 +1162,30 @@ vxorps (%rax){1to16}, %zmm17, %zmm19 {z}{k1} # CHECK-NEXT: - - 3.00 - 0.50 0.50 - - - - - - vpmullq (%rax), %zmm17, %zmm19 {%k1} # CHECK-NEXT: - - 3.00 - - - - - - - - - vpmullq %zmm16, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: - - 3.00 - 0.50 0.50 - - - - - - vpmullq (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangepd $ab, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangepd $ab, (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangeps $ab, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangeps $ab, (%rax), %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - 0.50 0.50 - 0.50 - - - - vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} -# CHECK-NEXT: - - 0.50 - - - - 0.50 - - - - vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangepd $ab, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangepd $ab, (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangepd $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangepd $ab, (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangepd $ab, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangepd $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangeps $ab, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangeps $ab, (%rax), %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangeps $ab, %zmm16, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangeps $ab, (%rax), %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - 0.50 0.50 - - - - - - vrangeps $ab, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} +# CHECK-NEXT: - - 1.00 - - - - - - - - - vrangeps $ab, {sae}, %zmm16, %zmm17, %zmm19 {%k1} {z} # CHECK-NEXT: - - 0.50 0.50 - - - - - - - - vrangesd $ab, %xmm16, %xmm17, %xmm19 # CHECK-NEXT: - - 0.50 0.50 0.50 0.50 - - - - - - vrangesd $ab, (%rax), %xmm17, %xmm19 # CHECK-NEXT: - - 0.50 0.50 - - - - - - - - vrangesd $ab, %xmm16, %xmm17, %xmm19 {%k1} -- GitLab From 070848c17c2944afa494d42d3ad42929f3379842 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 13 Feb 2024 09:29:56 +0100 Subject: [PATCH 027/284] [AArch64][GISel] Don't pointlessly lower G_TRUNC (#81479) If we have something like G_TRUNC from v2s32 to v2s16, then lowering this to a concat of two G_TRUNC s32 to s16 followed by G_TRUNC from v2s16 to v2s8 does not bring us any closer to legality. In fact, the first part of that is a G_BUILD_VECTOR whose legalization will produce a new G_TRUNC from v2s32 to v2s16, and both G_TRUNCs will then get combined to the original, causing a legalization cycle. Make the lowering condition more precise, by requiring that the original vector is >128 bits, which is I believe the only case where this specific splitting approach is useful. Note that this doesn't actually produce a legal result (the alwaysLegal is a lie, as before), but it will cause a proper globalisel abort instead of an infinite legalization loop. Fixes https://github.com/llvm/llvm-project/issues/81244. --- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 5 ++-- .../AArch64/GlobalISel/legalize-xtn.mir | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index ab25e2b9af8b..933f13dd5a19 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -612,9 +612,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .lowerIf([=](const LegalityQuery &Query) { LLT DstTy = Query.Types[0]; LLT SrcTy = Query.Types[1]; - return DstTy.isVector() && (SrcTy.getSizeInBits() > 128 || - (DstTy.getScalarSizeInBits() * 2 < - SrcTy.getScalarSizeInBits())); + return DstTy.isVector() && SrcTy.getSizeInBits() > 128 && + DstTy.getScalarSizeInBits() * 2 <= SrcTy.getScalarSizeInBits(); }) .alwaysLegal(); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir index 16b780a83973..661265173ae8 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir @@ -529,3 +529,27 @@ body: | RET_ReallyLR implicit $q0 ... + +--- +name: pr81244 +tracksRegLiveness: true +body: | + bb.0: + liveins: $d0 + ; CHECK-LABEL: name: pr81244 + ; CHECK: liveins: $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s8>) = G_TRUNC [[COPY]](<2 x s32>) + ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s8>) = G_CONCAT_VECTORS [[TRUNC]](<2 x s8>), [[TRUNC]](<2 x s8>) + ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[CONCAT_VECTORS]](<4 x s8>) + ; CHECK-NEXT: $d0 = COPY [[ANYEXT]](<4 x s16>) + ; CHECK-NEXT: RET_ReallyLR implicit $d0 + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s8>) = G_TRUNC %0(<2 x s32>) + %2:_(<4 x s8>) = G_CONCAT_VECTORS %1(<2 x s8>), %1(<2 x s8>) + %3:_(<4 x s16>) = G_ANYEXT %2(<4 x s8>) + $d0 = COPY %3(<4 x s16>) + RET_ReallyLR implicit $d0 + +... -- GitLab From 815a84655262ac569db11357fef1651f3571e7ee Mon Sep 17 00:00:00 2001 From: David Green Date: Tue, 13 Feb 2024 08:31:07 +0000 Subject: [PATCH 028/284] [Flang] Move genMinMaxlocReductionLoop to Transforms/Utils.cpp (#81380) This is one option for attempting to move genMinMaxlocReductionLoop to a better location. It moves it into Transforms and makes HLFIRTranforms depend upon FIRTransforms. It passes a build locally, both with and without -DBUILD_SHARED_LIBS, and does OK on the windows CI. --- flang/include/flang/Optimizer/Support/Utils.h | 139 ------------------ .../flang/Optimizer/Transforms/Utils.h | 38 +++++ flang/lib/Optimizer/Dialect/CMakeLists.txt | 1 - .../Optimizer/HLFIR/Transforms/CMakeLists.txt | 1 + .../Transforms/OptimizedBufferization.cpp | 2 +- .../Transforms/SimplifyIntrinsics.cpp | 131 ++++++++++++++++- 6 files changed, 170 insertions(+), 142 deletions(-) create mode 100644 flang/include/flang/Optimizer/Transforms/Utils.h diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h index 4e06bf8bafdc..7a8a34c25ce9 100644 --- a/flang/include/flang/Optimizer/Support/Utils.h +++ b/flang/include/flang/Optimizer/Support/Utils.h @@ -18,7 +18,6 @@ #include "flang/Optimizer/Builder/Todo.h" #include "flang/Optimizer/Dialect/FIROps.h" #include "flang/Optimizer/Dialect/FIRType.h" -#include "flang/Optimizer/HLFIR/HLFIRDialect.h" #include "flang/Optimizer/Support/FatalError.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Func/IR/FuncOps.h" @@ -135,144 +134,6 @@ inline void intrinsicTypeTODO(fir::FirOpBuilder &builder, mlir::Type type, " in " + intrinsicName); } -using MinlocBodyOpGeneratorTy = llvm::function_ref &)>; -using InitValGeneratorTy = llvm::function_ref; -using AddrGeneratorTy = llvm::function_ref; - -// Produces a loop nest for a Minloc intrinsic. -inline void genMinMaxlocReductionLoop( - fir::FirOpBuilder &builder, mlir::Value array, - fir::InitValGeneratorTy initVal, fir::MinlocBodyOpGeneratorTy genBody, - fir::AddrGeneratorTy getAddrFn, unsigned rank, mlir::Type elementType, - mlir::Location loc, mlir::Type maskElemType, mlir::Value resultArr, - bool maskMayBeLogicalScalar) { - mlir::IndexType idxTy = builder.getIndexType(); - - mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0); - - fir::SequenceType::Shape flatShape(rank, - fir::SequenceType::getUnknownExtent()); - mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType); - mlir::Type boxArrTy = fir::BoxType::get(arrTy); - array = builder.create(loc, boxArrTy, array); - - mlir::Type resultElemType = hlfir::getFortranElementType(resultArr.getType()); - mlir::Value flagSet = builder.createIntegerConstant(loc, resultElemType, 1); - mlir::Value zero = builder.createIntegerConstant(loc, resultElemType, 0); - mlir::Value flagRef = builder.createTemporary(loc, resultElemType); - builder.create(loc, zero, flagRef); - - mlir::Value init = initVal(builder, loc, elementType); - llvm::SmallVector bounds; - - assert(rank > 0 && "rank cannot be zero"); - mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1); - - // Compute all the upper bounds before the loop nest. - // It is not strictly necessary for performance, since the loop nest - // does not have any store operations and any LICM optimization - // should be able to optimize the redundancy. - for (unsigned i = 0; i < rank; ++i) { - mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i); - auto dims = - builder.create(loc, idxTy, idxTy, idxTy, array, dimIdx); - mlir::Value len = dims.getResult(1); - // We use C indexing here, so len-1 as loopcount - mlir::Value loopCount = builder.create(loc, len, one); - bounds.push_back(loopCount); - } - // Create a loop nest consisting of OP operations. - // Collect the loops' induction variables into indices array, - // which will be used in the innermost loop to load the input - // array's element. - // The loops are generated such that the innermost loop processes - // the 0 dimension. - llvm::SmallVector indices; - for (unsigned i = rank; 0 < i; --i) { - mlir::Value step = one; - mlir::Value loopCount = bounds[i - 1]; - auto loop = - builder.create(loc, zeroIdx, loopCount, step, false, - /*finalCountValue=*/false, init); - init = loop.getRegionIterArgs()[0]; - indices.push_back(loop.getInductionVar()); - // Set insertion point to the loop body so that the next loop - // is inserted inside the current one. - builder.setInsertionPointToStart(loop.getBody()); - } - - // Reverse the indices such that they are ordered as: - // - std::reverse(indices.begin(), indices.end()); - mlir::Value reductionVal = - genBody(builder, loc, elementType, array, flagRef, init, indices); - - // Unwind the loop nest and insert ResultOp on each level - // to return the updated value of the reduction to the enclosing - // loops. - for (unsigned i = 0; i < rank; ++i) { - auto result = builder.create(loc, reductionVal); - // Proceed to the outer loop. - auto loop = mlir::cast(result->getParentOp()); - reductionVal = loop.getResult(0); - // Set insertion point after the loop operation that we have - // just processed. - builder.setInsertionPointAfter(loop.getOperation()); - } - // End of loop nest. The insertion point is after the outermost loop. - if (maskMayBeLogicalScalar) { - if (fir::IfOp ifOp = - mlir::dyn_cast(builder.getBlock()->getParentOp())) { - builder.create(loc, reductionVal); - builder.setInsertionPointAfter(ifOp); - // Redefine flagSet to escape scope of ifOp - flagSet = builder.createIntegerConstant(loc, resultElemType, 1); - reductionVal = ifOp.getResult(0); - } - } - - // Check for case where array was full of max values. - // flag will be 0 if mask was never true, 1 if mask was true as some point, - // this is needed to avoid catching cases where we didn't access any elements - // e.g. mask=.FALSE. - mlir::Value flagValue = - builder.create(loc, resultElemType, flagRef); - mlir::Value flagCmp = builder.create( - loc, mlir::arith::CmpIPredicate::eq, flagValue, flagSet); - fir::IfOp ifMaskTrueOp = - builder.create(loc, flagCmp, /*withElseRegion=*/false); - builder.setInsertionPointToStart(&ifMaskTrueOp.getThenRegion().front()); - - mlir::Value testInit = initVal(builder, loc, elementType); - fir::IfOp ifMinSetOp; - if (elementType.isa()) { - mlir::Value cmp = builder.create( - loc, mlir::arith::CmpFPredicate::OEQ, testInit, reductionVal); - ifMinSetOp = builder.create(loc, cmp, - /*withElseRegion*/ false); - } else { - mlir::Value cmp = builder.create( - loc, mlir::arith::CmpIPredicate::eq, testInit, reductionVal); - ifMinSetOp = builder.create(loc, cmp, - /*withElseRegion*/ false); - } - builder.setInsertionPointToStart(&ifMinSetOp.getThenRegion().front()); - - // Load output array with 1s instead of 0s - for (unsigned int i = 0; i < rank; ++i) { - mlir::Value index = builder.createIntegerConstant(loc, idxTy, i); - mlir::Value resultElemAddr = - getAddrFn(builder, loc, resultElemType, resultArr, index); - builder.create(loc, flagSet, resultElemAddr); - } - builder.setInsertionPointAfter(ifMaskTrueOp); -} - inline fir::CUDADataAttributeAttr getCUDADataAttribute(mlir::MLIRContext *mlirContext, std::optional cudaAttr) { diff --git a/flang/include/flang/Optimizer/Transforms/Utils.h b/flang/include/flang/Optimizer/Transforms/Utils.h new file mode 100644 index 000000000000..49a616fb40fd --- /dev/null +++ b/flang/include/flang/Optimizer/Transforms/Utils.h @@ -0,0 +1,38 @@ +//===-- Optimizer/Transforms/Utils.h ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ +// +//===----------------------------------------------------------------------===// + +#ifndef FORTRAN_OPTIMIZER_TRANSFORMS_UTILS_H +#define FORTRAN_OPTIMIZER_TRANSFORMS_UTILS_H + +namespace fir { + +using MinlocBodyOpGeneratorTy = llvm::function_ref &)>; +using InitValGeneratorTy = llvm::function_ref; +using AddrGeneratorTy = llvm::function_ref; + +// Produces a loop nest for a Minloc intrinsic. +void genMinMaxlocReductionLoop(fir::FirOpBuilder &builder, mlir::Value array, + fir::InitValGeneratorTy initVal, + fir::MinlocBodyOpGeneratorTy genBody, + fir::AddrGeneratorTy getAddrFn, unsigned rank, + mlir::Type elementType, mlir::Location loc, + mlir::Type maskElemType, mlir::Value resultArr, + bool maskMayBeLogicalScalar); + +} // namespace fir + +#endif // FORTRAN_OPTIMIZER_TRANSFORMS_UTILS_H diff --git a/flang/lib/Optimizer/Dialect/CMakeLists.txt b/flang/lib/Optimizer/Dialect/CMakeLists.txt index 58a4276224b3..745439b7e1e5 100644 --- a/flang/lib/Optimizer/Dialect/CMakeLists.txt +++ b/flang/lib/Optimizer/Dialect/CMakeLists.txt @@ -13,7 +13,6 @@ add_flang_library(FIRDialect CanonicalizationPatternsIncGen MLIRIR FIROpsIncGen - HLFIROpsIncGen intrinsics_gen LINK_LIBS diff --git a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt index 603b328a6823..ad569ce3b41f 100644 --- a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt +++ b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt @@ -21,6 +21,7 @@ add_flang_library(HLFIRTransforms FIRBuilder FIRDialectSupport FIRSupport + FIRTransforms HLFIRDialect MLIRIR ${dialect_libs} diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp index 523671f0605f..c2512c7df32f 100644 --- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp +++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp @@ -20,7 +20,7 @@ #include "flang/Optimizer/HLFIR/HLFIRDialect.h" #include "flang/Optimizer/HLFIR/HLFIROps.h" #include "flang/Optimizer/HLFIR/Passes.h" -#include "flang/Optimizer/Support/Utils.h" +#include "flang/Optimizer/Transforms/Utils.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/IR/Dominance.h" #include "mlir/IR/PatternMatch.h" diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp index b415463075d6..86343e23c6e5 100644 --- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp +++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp @@ -31,8 +31,8 @@ #include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Optimizer/Dialect/Support/FIRContext.h" #include "flang/Optimizer/HLFIR/HLFIRDialect.h" -#include "flang/Optimizer/Support/Utils.h" #include "flang/Optimizer/Transforms/Passes.h" +#include "flang/Optimizer/Transforms/Utils.h" #include "flang/Runtime/entry-names.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/IR/Matchers.h" @@ -558,6 +558,135 @@ static mlir::FunctionType genRuntimeMinlocType(fir::FirOpBuilder &builder, {boxRefType, boxType, boxType}, {}); } +// Produces a loop nest for a Minloc intrinsic. +void fir::genMinMaxlocReductionLoop( + fir::FirOpBuilder &builder, mlir::Value array, + fir::InitValGeneratorTy initVal, fir::MinlocBodyOpGeneratorTy genBody, + fir::AddrGeneratorTy getAddrFn, unsigned rank, mlir::Type elementType, + mlir::Location loc, mlir::Type maskElemType, mlir::Value resultArr, + bool maskMayBeLogicalScalar) { + mlir::IndexType idxTy = builder.getIndexType(); + + mlir::Value zeroIdx = builder.createIntegerConstant(loc, idxTy, 0); + + fir::SequenceType::Shape flatShape(rank, + fir::SequenceType::getUnknownExtent()); + mlir::Type arrTy = fir::SequenceType::get(flatShape, elementType); + mlir::Type boxArrTy = fir::BoxType::get(arrTy); + array = builder.create(loc, boxArrTy, array); + + mlir::Type resultElemType = hlfir::getFortranElementType(resultArr.getType()); + mlir::Value flagSet = builder.createIntegerConstant(loc, resultElemType, 1); + mlir::Value zero = builder.createIntegerConstant(loc, resultElemType, 0); + mlir::Value flagRef = builder.createTemporary(loc, resultElemType); + builder.create(loc, zero, flagRef); + + mlir::Value init = initVal(builder, loc, elementType); + llvm::SmallVector bounds; + + assert(rank > 0 && "rank cannot be zero"); + mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1); + + // Compute all the upper bounds before the loop nest. + // It is not strictly necessary for performance, since the loop nest + // does not have any store operations and any LICM optimization + // should be able to optimize the redundancy. + for (unsigned i = 0; i < rank; ++i) { + mlir::Value dimIdx = builder.createIntegerConstant(loc, idxTy, i); + auto dims = + builder.create(loc, idxTy, idxTy, idxTy, array, dimIdx); + mlir::Value len = dims.getResult(1); + // We use C indexing here, so len-1 as loopcount + mlir::Value loopCount = builder.create(loc, len, one); + bounds.push_back(loopCount); + } + // Create a loop nest consisting of OP operations. + // Collect the loops' induction variables into indices array, + // which will be used in the innermost loop to load the input + // array's element. + // The loops are generated such that the innermost loop processes + // the 0 dimension. + llvm::SmallVector indices; + for (unsigned i = rank; 0 < i; --i) { + mlir::Value step = one; + mlir::Value loopCount = bounds[i - 1]; + auto loop = + builder.create(loc, zeroIdx, loopCount, step, false, + /*finalCountValue=*/false, init); + init = loop.getRegionIterArgs()[0]; + indices.push_back(loop.getInductionVar()); + // Set insertion point to the loop body so that the next loop + // is inserted inside the current one. + builder.setInsertionPointToStart(loop.getBody()); + } + + // Reverse the indices such that they are ordered as: + // + std::reverse(indices.begin(), indices.end()); + mlir::Value reductionVal = + genBody(builder, loc, elementType, array, flagRef, init, indices); + + // Unwind the loop nest and insert ResultOp on each level + // to return the updated value of the reduction to the enclosing + // loops. + for (unsigned i = 0; i < rank; ++i) { + auto result = builder.create(loc, reductionVal); + // Proceed to the outer loop. + auto loop = mlir::cast(result->getParentOp()); + reductionVal = loop.getResult(0); + // Set insertion point after the loop operation that we have + // just processed. + builder.setInsertionPointAfter(loop.getOperation()); + } + // End of loop nest. The insertion point is after the outermost loop. + if (maskMayBeLogicalScalar) { + if (fir::IfOp ifOp = + mlir::dyn_cast(builder.getBlock()->getParentOp())) { + builder.create(loc, reductionVal); + builder.setInsertionPointAfter(ifOp); + // Redefine flagSet to escape scope of ifOp + flagSet = builder.createIntegerConstant(loc, resultElemType, 1); + reductionVal = ifOp.getResult(0); + } + } + + // Check for case where array was full of max values. + // flag will be 0 if mask was never true, 1 if mask was true as some point, + // this is needed to avoid catching cases where we didn't access any elements + // e.g. mask=.FALSE. + mlir::Value flagValue = + builder.create(loc, resultElemType, flagRef); + mlir::Value flagCmp = builder.create( + loc, mlir::arith::CmpIPredicate::eq, flagValue, flagSet); + fir::IfOp ifMaskTrueOp = + builder.create(loc, flagCmp, /*withElseRegion=*/false); + builder.setInsertionPointToStart(&ifMaskTrueOp.getThenRegion().front()); + + mlir::Value testInit = initVal(builder, loc, elementType); + fir::IfOp ifMinSetOp; + if (elementType.isa()) { + mlir::Value cmp = builder.create( + loc, mlir::arith::CmpFPredicate::OEQ, testInit, reductionVal); + ifMinSetOp = builder.create(loc, cmp, + /*withElseRegion*/ false); + } else { + mlir::Value cmp = builder.create( + loc, mlir::arith::CmpIPredicate::eq, testInit, reductionVal); + ifMinSetOp = builder.create(loc, cmp, + /*withElseRegion*/ false); + } + builder.setInsertionPointToStart(&ifMinSetOp.getThenRegion().front()); + + // Load output array with 1s instead of 0s + for (unsigned int i = 0; i < rank; ++i) { + mlir::Value index = builder.createIntegerConstant(loc, idxTy, i); + mlir::Value resultElemAddr = + getAddrFn(builder, loc, resultElemType, resultArr, index); + builder.create(loc, flagSet, resultElemAddr); + } + builder.setInsertionPointAfter(ifMaskTrueOp); +} + static void genRuntimeMinMaxlocBody(fir::FirOpBuilder &builder, mlir::func::FuncOp &funcOp, bool isMax, unsigned rank, int maskRank, -- GitLab From f0db35b93f31ea5d6ff9bd4791fb6755b5a5bb9b Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Tue, 13 Feb 2024 17:40:51 +0900 Subject: [PATCH 029/284] [MC/DC] Refactor: Introduce `MCDCTypes.h` for `coverage::mcdc` (#81459) They can be also used in `clang`. Introduce the lightweight header instead of `CoverageMapping.h`. This includes for now: * `mcdc::ConditionID` * `mcdc::Parameters` --- clang/lib/CodeGen/CoverageMappingGen.cpp | 54 +++++++++---------- .../ProfileData/Coverage/CoverageMapping.h | 33 ++++-------- .../llvm/ProfileData/Coverage/MCDCTypes.h | 36 +++++++++++++ .../ProfileData/Coverage/CoverageMapping.cpp | 2 +- .../Coverage/CoverageMappingReader.cpp | 7 ++- .../ProfileData/CoverageMappingTest.cpp | 7 ++- 6 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp index 0c43317642bc..5060b5b2860f 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -95,9 +95,6 @@ void CoverageSourceInfo::updateNextTokLoc(SourceLocation Loc) { } namespace { -using MCDCConditionID = CounterMappingRegion::MCDCConditionID; -using MCDCParameters = CounterMappingRegion::MCDCParameters; - /// A region of source code that can be mapped to a counter. class SourceMappingRegion { /// Primary Counter that is also used for Branch Regions for "True" branches. @@ -107,7 +104,7 @@ class SourceMappingRegion { std::optional FalseCount; /// Parameters used for Modified Condition/Decision Coverage - MCDCParameters MCDCParams; + mcdc::Parameters MCDCParams; /// The region's starting location. std::optional LocStart; @@ -131,7 +128,7 @@ public: SkippedRegion(false) {} SourceMappingRegion(Counter Count, std::optional FalseCount, - MCDCParameters MCDCParams, + mcdc::Parameters MCDCParams, std::optional LocStart, std::optional LocEnd, bool GapRegion = false) @@ -139,7 +136,7 @@ public: LocStart(LocStart), LocEnd(LocEnd), GapRegion(GapRegion), SkippedRegion(false) {} - SourceMappingRegion(MCDCParameters MCDCParams, + SourceMappingRegion(mcdc::Parameters MCDCParams, std::optional LocStart, std::optional LocEnd) : MCDCParams(MCDCParams), LocStart(LocStart), LocEnd(LocEnd), @@ -187,7 +184,7 @@ public: bool isMCDCDecision() const { return MCDCParams.NumConditions != 0; } - const MCDCParameters &getMCDCParams() const { return MCDCParams; } + const mcdc::Parameters &getMCDCParams() const { return MCDCParams; } }; /// Spelling locations for the start and end of a source region. @@ -587,8 +584,8 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder { struct MCDCCoverageBuilder { struct DecisionIDPair { - MCDCConditionID TrueID = 0; - MCDCConditionID FalseID = 0; + mcdc::ConditionID TrueID = 0; + mcdc::ConditionID FalseID = 0; }; /// The AST walk recursively visits nested logical-AND or logical-OR binary @@ -682,9 +679,9 @@ private: CodeGenModule &CGM; llvm::SmallVector DecisionStack; - llvm::DenseMap &CondIDs; + llvm::DenseMap &CondIDs; llvm::DenseMap &MCDCBitmapMap; - MCDCConditionID NextID = 1; + mcdc::ConditionID NextID = 1; bool NotMapped = false; /// Represent a sentinel value of [0,0] for the bottom of DecisionStack. @@ -696,9 +693,10 @@ private: } public: - MCDCCoverageBuilder(CodeGenModule &CGM, - llvm::DenseMap &CondIDMap, - llvm::DenseMap &MCDCBitmapMap) + MCDCCoverageBuilder( + CodeGenModule &CGM, + llvm::DenseMap &CondIDMap, + llvm::DenseMap &MCDCBitmapMap) : CGM(CGM), DecisionStack(1, DecisionStackSentinel), CondIDs(CondIDMap), MCDCBitmapMap(MCDCBitmapMap) {} @@ -713,12 +711,12 @@ public: bool isBuilding() const { return (NextID > 1); } /// Set the given condition's ID. - void setCondID(const Expr *Cond, MCDCConditionID ID) { + void setCondID(const Expr *Cond, mcdc::ConditionID ID) { CondIDs[CodeGenFunction::stripCond(Cond)] = ID; } /// Return the ID of a given condition. - MCDCConditionID getCondID(const Expr *Cond) const { + mcdc::ConditionID getCondID(const Expr *Cond) const { auto I = CondIDs.find(CodeGenFunction::stripCond(Cond)); if (I == CondIDs.end()) return 0; @@ -755,7 +753,7 @@ public: setCondID(E->getLHS(), NextID++); // Assign a ID+1 for the RHS. - MCDCConditionID RHSid = NextID++; + mcdc::ConditionID RHSid = NextID++; setCondID(E->getRHS(), RHSid); // Push the LHS decision IDs onto the DecisionStack. @@ -865,8 +863,8 @@ struct CounterCoverageMappingBuilder std::optional StartLoc = std::nullopt, std::optional EndLoc = std::nullopt, std::optional FalseCount = std::nullopt, - MCDCConditionID ID = 0, MCDCConditionID TrueID = 0, - MCDCConditionID FalseID = 0) { + mcdc::ConditionID ID = 0, mcdc::ConditionID TrueID = 0, + mcdc::ConditionID FalseID = 0) { if (StartLoc && !FalseCount) { MostRecentLocation = *StartLoc; @@ -886,7 +884,7 @@ struct CounterCoverageMappingBuilder if (EndLoc && EndLoc->isInvalid()) EndLoc = std::nullopt; RegionStack.emplace_back(Count, FalseCount, - MCDCParameters{0, 0, ID, TrueID, FalseID}, + mcdc::Parameters{0, 0, ID, TrueID, FalseID}, StartLoc, EndLoc); return RegionStack.size() - 1; @@ -896,7 +894,7 @@ struct CounterCoverageMappingBuilder std::optional StartLoc = std::nullopt, std::optional EndLoc = std::nullopt) { - RegionStack.emplace_back(MCDCParameters{BitmapIdx, Conditions}, StartLoc, + RegionStack.emplace_back(mcdc::Parameters{BitmapIdx, Conditions}, StartLoc, EndLoc); return RegionStack.size() - 1; @@ -1042,9 +1040,9 @@ struct CounterCoverageMappingBuilder // function's SourceRegions) because it doesn't apply to any other source // code other than the Condition. if (CodeGenFunction::isInstrumentedCondition(C)) { - MCDCConditionID ID = MCDCBuilder.getCondID(C); - MCDCConditionID TrueID = IDPair.TrueID; - MCDCConditionID FalseID = IDPair.FalseID; + mcdc::ConditionID ID = MCDCBuilder.getCondID(C); + mcdc::ConditionID TrueID = IDPair.TrueID; + mcdc::ConditionID FalseID = IDPair.FalseID; // If a condition can fold to true or false, the corresponding branch // will be removed. Create a region with both counters hard-coded to @@ -1151,9 +1149,9 @@ struct CounterCoverageMappingBuilder if (I.isBranch()) SourceRegions.emplace_back( I.getCounter(), I.getFalseCounter(), - MCDCParameters{0, 0, I.getMCDCParams().ID, - I.getMCDCParams().TrueID, - I.getMCDCParams().FalseID}, + mcdc::Parameters{0, 0, I.getMCDCParams().ID, + I.getMCDCParams().TrueID, + I.getMCDCParams().FalseID}, Loc, getEndOfFileOrMacro(Loc), I.isBranch()); else SourceRegions.emplace_back(I.getCounter(), Loc, @@ -1338,7 +1336,7 @@ struct CounterCoverageMappingBuilder CoverageMappingModuleGen &CVM, llvm::DenseMap &CounterMap, llvm::DenseMap &MCDCBitmapMap, - llvm::DenseMap &CondIDMap, + llvm::DenseMap &CondIDMap, SourceManager &SM, const LangOptions &LangOpts) : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap), MCDCBitmapMap(MCDCBitmapMap), diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 88ec60c7aa33..2869ae8fe861 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -23,6 +23,7 @@ #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Object/BuildID.h" +#include "llvm/ProfileData/Coverage/MCDCTypes.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/Compiler.h" @@ -249,19 +250,6 @@ struct CounterMappingRegion { MCDCBranchRegion }; - using MCDCConditionID = unsigned int; - struct MCDCParameters { - /// Byte Index of Bitmap Coverage Object for a Decision Region. - unsigned BitmapIdx = 0; - - /// Number of Conditions used for a Decision Region. - unsigned NumConditions = 0; - - /// IDs used to represent a branch region and other branch regions - /// evaluated based on True and False branches. - MCDCConditionID ID = 0, TrueID = 0, FalseID = 0; - }; - /// Primary Counter that is also used for Branch Regions (TrueCount). Counter Count; @@ -269,7 +257,7 @@ struct CounterMappingRegion { Counter FalseCount; /// Parameters used for Modified Condition/Decision Coverage - MCDCParameters MCDCParams; + mcdc::Parameters MCDCParams; unsigned FileID = 0; unsigned ExpandedFileID = 0; @@ -285,7 +273,7 @@ struct CounterMappingRegion { ColumnEnd(ColumnEnd), Kind(Kind) {} CounterMappingRegion(Counter Count, Counter FalseCount, - MCDCParameters MCDCParams, unsigned FileID, + mcdc::Parameters MCDCParams, unsigned FileID, unsigned ExpandedFileID, unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd, RegionKind Kind) @@ -294,7 +282,7 @@ struct CounterMappingRegion { ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd), Kind(Kind) {} - CounterMappingRegion(MCDCParameters MCDCParams, unsigned FileID, + CounterMappingRegion(mcdc::Parameters MCDCParams, unsigned FileID, unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd, RegionKind Kind) : MCDCParams(MCDCParams), FileID(FileID), LineStart(LineStart), @@ -334,15 +322,16 @@ struct CounterMappingRegion { makeBranchRegion(Counter Count, Counter FalseCount, unsigned FileID, unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) { - return CounterMappingRegion(Count, FalseCount, MCDCParameters(), FileID, 0, - LineStart, ColumnStart, LineEnd, ColumnEnd, + return CounterMappingRegion(Count, FalseCount, mcdc::Parameters(), FileID, + 0, LineStart, ColumnStart, LineEnd, ColumnEnd, BranchRegion); } static CounterMappingRegion - makeBranchRegion(Counter Count, Counter FalseCount, MCDCParameters MCDCParams, - unsigned FileID, unsigned LineStart, unsigned ColumnStart, - unsigned LineEnd, unsigned ColumnEnd) { + makeBranchRegion(Counter Count, Counter FalseCount, + mcdc::Parameters MCDCParams, unsigned FileID, + unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, + unsigned ColumnEnd) { return CounterMappingRegion(Count, FalseCount, MCDCParams, FileID, 0, LineStart, ColumnStart, LineEnd, ColumnEnd, MCDCParams.ID == 0 ? BranchRegion @@ -350,7 +339,7 @@ struct CounterMappingRegion { } static CounterMappingRegion - makeDecisionRegion(MCDCParameters MCDCParams, unsigned FileID, + makeDecisionRegion(mcdc::Parameters MCDCParams, unsigned FileID, unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) { return CounterMappingRegion(MCDCParams, FileID, LineStart, ColumnStart, diff --git a/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h b/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h new file mode 100644 index 000000000000..365c899b6eb5 --- /dev/null +++ b/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h @@ -0,0 +1,36 @@ +//===- MCDCTypes.h - Types related to MC/DC Coverage ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Types related to MC/DC Coverage. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_PROFILEDATA_COVERAGE_MCDCTYPES_H +#define LLVM_PROFILEDATA_COVERAGE_MCDCTYPES_H + +namespace llvm::coverage::mcdc { + +/// The ID for MCDCBranch. +using ConditionID = unsigned int; + +/// MC/DC-specifig parameters +struct Parameters { + /// Byte Index of Bitmap Coverage Object for a Decision Region. + unsigned BitmapIdx = 0; + + /// Number of Conditions used for a Decision Region. + unsigned NumConditions = 0; + + /// IDs used to represent a branch region and other branch regions + /// evaluated based on True and False branches. + ConditionID ID = 0, TrueID = 0, FalseID = 0; +}; + +} // namespace llvm::coverage::mcdc + +#endif // LLVM_PROFILEDATA_COVERAGE_MCDCTYPES_H diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index eb0996e33b70..517a81d57401 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -524,7 +524,7 @@ private: /// IDs that are stored in MCDCBranches /// Complete when all IDs (1 to NumConditions) are met. - DenseSet ConditionIDs; + DenseSet ConditionIDs; /// Set of IDs of Expansion(s) that are relevant to DecisionRegion /// and its children (via expansions). diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index ac8e6b56379f..fc6014c1f7be 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -371,10 +371,9 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray( auto CMR = CounterMappingRegion( C, C2, - CounterMappingRegion::MCDCParameters{ - static_cast(BIDX), static_cast(NC), - static_cast(ID), static_cast(TID), - static_cast(FID)}, + mcdc::Parameters{static_cast(BIDX), static_cast(NC), + static_cast(ID), static_cast(TID), + static_cast(FID)}, InferredFileID, ExpandedFileID, LineStart, ColumnStart, LineStart + NumLines, ColumnEnd, Kind); if (CMR.startLoc() > CMR.endLoc()) diff --git a/llvm/unittests/ProfileData/CoverageMappingTest.cpp b/llvm/unittests/ProfileData/CoverageMappingTest.cpp index 2849781a9dc4..d60312b68237 100644 --- a/llvm/unittests/ProfileData/CoverageMappingTest.cpp +++ b/llvm/unittests/ProfileData/CoverageMappingTest.cpp @@ -197,8 +197,7 @@ struct CoverageMappingTest : ::testing::TestWithParam> { auto &Regions = InputFunctions.back().Regions; unsigned FileID = getFileIndexForFunction(File); Regions.push_back(CounterMappingRegion::makeDecisionRegion( - CounterMappingRegion::MCDCParameters{Mask, NC}, FileID, LS, CS, LE, - CE)); + mcdc::Parameters{Mask, NC}, FileID, LS, CS, LE, CE)); } void addMCDCBranchCMR(Counter C1, Counter C2, unsigned ID, unsigned TrueID, @@ -207,8 +206,8 @@ struct CoverageMappingTest : ::testing::TestWithParam> { auto &Regions = InputFunctions.back().Regions; unsigned FileID = getFileIndexForFunction(File); Regions.push_back(CounterMappingRegion::makeBranchRegion( - C1, C2, CounterMappingRegion::MCDCParameters{0, 0, ID, TrueID, FalseID}, - FileID, LS, CS, LE, CE)); + C1, C2, mcdc::Parameters{0, 0, ID, TrueID, FalseID}, FileID, LS, CS, LE, + CE)); } void addExpansionCMR(StringRef File, StringRef ExpandedFile, unsigned LS, -- GitLab From 05ad0d46325732e2f7759cb93c94f3e15b41d110 Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Tue, 13 Feb 2024 17:45:28 +0900 Subject: [PATCH 030/284] CoverageMapping.cpp: Apply std::move to MCDCRecord (#81220) --- .../llvm/ProfileData/Coverage/CoverageMapping.h | 16 ++++++++++------ .../lib/ProfileData/Coverage/CoverageMapping.cpp | 8 ++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 2869ae8fe861..e877571779fa 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -396,11 +396,13 @@ private: LineColPairMap CondLoc; public: - MCDCRecord(CounterMappingRegion Region, TestVectors TV, - TVPairMap IndependencePairs, BoolVector Folded, CondIDMap PosToID, - LineColPairMap CondLoc) - : Region(Region), TV(TV), IndependencePairs(IndependencePairs), - Folded(Folded), PosToID(PosToID), CondLoc(CondLoc){}; + MCDCRecord(const CounterMappingRegion &Region, TestVectors &&TV, + TVPairMap &&IndependencePairs, BoolVector &&Folded, + CondIDMap &&PosToID, LineColPairMap &&CondLoc) + : Region(Region), TV(std::move(TV)), + IndependencePairs(std::move(IndependencePairs)), + Folded(std::move(Folded)), PosToID(std::move(PosToID)), + CondLoc(std::move(CondLoc)){}; CounterMappingRegion getDecisionRegion() const { return Region; } unsigned getNumConditions() const { @@ -603,7 +605,9 @@ struct FunctionRecord { FunctionRecord(FunctionRecord &&FR) = default; FunctionRecord &operator=(FunctionRecord &&) = default; - void pushMCDCRecord(MCDCRecord Record) { MCDCRecords.push_back(Record); } + void pushMCDCRecord(MCDCRecord &&Record) { + MCDCRecords.push_back(std::move(Record)); + } void pushRegion(CounterMappingRegion Region, uint64_t Count, uint64_t FalseCount) { diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp index 517a81d57401..0c65ab8ce0dd 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp @@ -385,9 +385,9 @@ public: findIndependencePairs(); // Record Test vectors, executed vectors, and independence pairs. - MCDCRecord Res(Region, ExecVectors, IndependencePairs, Folded, PosToID, - CondLoc); - return Res; + return MCDCRecord(Region, std::move(ExecVectors), + std::move(IndependencePairs), std::move(Folded), + std::move(PosToID), std::move(CondLoc)); } }; @@ -761,7 +761,7 @@ Error CoverageMapping::loadFunctionRecord( } // Save the MC/DC Record so that it can be visualized later. - Function.pushMCDCRecord(*Record); + Function.pushMCDCRecord(std::move(*Record)); } // Don't create records for (filenames, function) pairs we've already seen. -- GitLab From 0a600c34c8c1fe87c9661b6020e5044b24da3dc7 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Tue, 13 Feb 2024 09:50:34 +0100 Subject: [PATCH 031/284] [mlir][nvgpu] Make `phaseParity` of `mbarrier.try_wait` `i1` (#81460) Currently, `phaseParity` argument of `nvgpu.mbarrier.try_wait.parity` is index. This can cause a problem if it's passed any value different than 0 or 1. Because the PTX instruction only accepts even or odd phase. This PR makes phaseParity argument i1 to avoid misuse. Here is the information from PTX doc: ``` The .parity variant of the instructions test for the completion of the phase indicated by the operand phaseParity, which is the integer parity of either the current phase or the immediately preceding phase of the mbarrier object. An even phase has integer parity 0 and an odd phase has integer parity of 1. So the valid values of phaseParity operand are 0 and 1. ``` See for more information: https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait --- mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td | 10 ++++++---- mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 3 ++- .../Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp | 3 ++- mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir | 8 ++++---- mlir/test/Dialect/NVGPU/tmaload-transform.mlir | 2 +- .../GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir | 3 ++- .../CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir | 3 ++- .../GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir | 3 ++- .../GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir | 3 ++- .../GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir | 3 ++- 10 files changed, 25 insertions(+), 16 deletions(-) diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td index a0c0d4cfd871..dda8f31e688f 100644 --- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td +++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td @@ -609,14 +609,16 @@ def NVGPU_MBarrierTryWaitParityOp : NVGPU_Op<"mbarrier.try_wait.parity", []> { phase. Suspended thread resumes execution when the specified phase completes OR before the phase completes following a system-dependent time limit. + The `$phaseParity` specifies either even phase (0) or odd phase (1) to + wait. + Example: ```mlir - nvgpu.mbarrier.try_wait.parity %barrier, %phase, %ticks : !nvgpu.mbarrier.barrier> + nvgpu.mbarrier.try_wait.parity %barrier, %phaseParity, %ticks : !nvgpu.mbarrier.barrier> ``` - }]; - let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$phase, Index:$ticks, Index:$mbarId); - let assemblyFormat = "$barriers `[` $mbarId `]` `,` $phase `,` $ticks attr-dict `:` type($barriers)"; + let arguments = (ins NVGPU_MBarrierGroup:$barriers, I1:$phaseParity, Index:$ticks, Index:$mbarId); + let assemblyFormat = "$barriers `[` $mbarId `]` `,` $phaseParity `,` $ticks attr-dict `:` type($barriers)"; } def NVGPU_TmaPrefetchOp : NVGPU_Op<"tma.prefetch.descriptor", []> { diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp index 5080956a4589..9b5d19ebd783 100644 --- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp +++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp @@ -956,7 +956,8 @@ struct NVGPUMBarrierTryWaitParityLowering getMbarrierPtr(b, op.getBarriers().getType(), adaptor.getBarriers(), adaptor.getMbarId(), rewriter); Value ticks = truncToI32(b, adaptor.getTicks()); - Value phase = truncToI32(b, adaptor.getPhase()); + Value phase = + b.create(b.getI32Type(), adaptor.getPhaseParity()); if (isMbarrierShared(op.getBarriers().getType())) { rewriter.replaceOpWithNewOp( diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp index c81742233e6d..1635297a5447 100644 --- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp +++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp @@ -1010,7 +1010,8 @@ void HopperBuilder::buildBarrierArriveTx( void HopperBuilder::buildTryWaitParity( TypedValue barrier) { - Value parity = rewriter.create(loc, 0); + Type i1 = rewriter.getI1Type(); + Value parity = rewriter.create(loc, i1, 0); // 10M is an arbitrary, not too small or too big number to specify the number // of ticks before retry. // TODO: hoist this in a default dialect constant. diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir index 09a873fa331d..dbf8ead49f78 100644 --- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir +++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir @@ -590,12 +590,12 @@ func.func @mbarrier_txcount() { } - %phase = arith.constant 0 : index + %phase_c0 = arith.constant 0 : i1 %ticks = arith.constant 10000000 : index // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64 // CHECK: nvvm.mbarrier.try_wait.parity.shared %[[barPtr3]] - nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase, %ticks : !barrierType + nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase_c0, %ticks : !barrierType func.return } @@ -626,12 +626,12 @@ func.func @mbarrier_txcount_pred() { // CHECK: nvvm.mbarrier.arrive.expect_tx.shared %[[barPtr2]], {{.*}}, predicate = %[[P]] nvgpu.mbarrier.arrive.expect_tx %barrier[%c0], %txcount, predicate = %pred : !barrierType - %phase = arith.constant 0 : index + %phase_c0 = arith.constant 0 : i1 %ticks = arith.constant 10000000 : index // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64 // CHECK: nvvm.mbarrier.try_wait.parity.shared %[[barPtr3]] - nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase, %ticks : !barrierType + nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase_c0, %ticks : !barrierType func.return } diff --git a/mlir/test/Dialect/NVGPU/tmaload-transform.mlir b/mlir/test/Dialect/NVGPU/tmaload-transform.mlir index 29e300a992d3..40acd82cd055 100644 --- a/mlir/test/Dialect/NVGPU/tmaload-transform.mlir +++ b/mlir/test/Dialect/NVGPU/tmaload-transform.mlir @@ -62,7 +62,7 @@ func.func @main() { // CHECK: nvgpu.mbarrier.arrive.expect_tx %[[B]][%{{.*}}], %[[c0_7]] : // CHECK: } // - // CHECK: %[[c0_6:.*]] = arith.constant 0 : index + // CHECK: %[[c0_6:.*]] = llvm.mlir.constant(false) : i1 // CHECK: %[[c10000000:.*]] = arith.constant 10000000 : index // CHECK: nvgpu.mbarrier.try_wait.parity %[[B]][%{{.*}}], %[[c0_6]], %[[c10000000]] : diff --git a/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir b/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir index 35ca0ee8677c..51bcf459d83d 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir @@ -197,7 +197,8 @@ func.func @main() { { %ticks = arith.constant 10000000 : index // TMA wait - nvgpu.mbarrier.try_wait.parity %barrier[%i], %c0, %ticks : !barrierType + %phase_c0 = arith.constant 0 : i1 + nvgpu.mbarrier.try_wait.parity %barrier[%i], %phase_c0, %ticks : !barrierType %lhsSlice = memref.subview %lhsShmem [%i, 0, 0][1, 128, 64][1, 1, 1] : memref<2x128x64xf16, 3> to memref<128x64xf16, strided<[64, 1], offset: ?>, 3> %rhsSlice = memref.subview %rhsShmem [%i, 0, 0][1, 64, 128][1, 1, 1] : memref<2x64x128xf16, strided<[8192, 128, 1], offset: 16384>, 3> to memref<64x128xf16, strided<[128, 1], offset: ?>, 3> // Descriptor WGMMA diff --git a/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir b/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir index 5a10bbba26d8..85bdb38d67f0 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir @@ -206,7 +206,8 @@ func.func @main() { { %ticks = arith.constant 10000000 : index // TMA wait - nvgpu.mbarrier.try_wait.parity %barrier[%i], %c0, %ticks : !barrierType + %phase_c0 = arith.constant 0 : i1 + nvgpu.mbarrier.try_wait.parity %barrier[%i], %phase_c0, %ticks : !barrierType %lhsSlice = memref.subview %lhsShmem [%i, 0, 0][1, 128, 64][1, 1, 1] : memref<2x128x64xf16, 3> to memref<128x64xf16, strided<[64, 1], offset: ?>, 3> %rhsSlice = memref.subview %rhsShmem [%i, 0, 0][1, 64, 128][1, 1, 1] : memref<2x64x128xf16, strided<[8192, 128, 1], offset: 16384>, 3> to memref<64x128xf16, strided<[128, 1], offset: ?>, 3> // Descriptor WGMMA diff --git a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir index 9c5aacf96b0d..b50772f8249f 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir @@ -93,7 +93,8 @@ module @mymod { } // Step 8. Wait until TMA is done - nvgpu.mbarrier.try_wait.parity %9[%c0], %c0, %c10000000 : !barrierType + %phase_c0 = arith.constant 0 : i1 + nvgpu.mbarrier.try_wait.parity %9[%c0], %phase_c0, %c10000000 : !barrierType // Step 9. Print loaded data in 128b swizzled scf.if %10 { diff --git a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir index 536e71d260f5..65e5fc0aff6a 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir @@ -119,7 +119,8 @@ module @mymod { } // Step 7. Wait until TMA is done - nvgpu.mbarrier.try_wait.parity %9[%c0], %c0, %c10000000 : !barrierType + %phase_c0 = arith.constant 0 : i1 + nvgpu.mbarrier.try_wait.parity %9[%c0], %phase_c0, %c10000000 : !barrierType // Step 8. Print loaded data in 128b swizzled scf.if %10 { diff --git a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir index aee265e3faf1..2e59b7234e53 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir @@ -96,7 +96,8 @@ module @mymod { } else { nvgpu.mbarrier.arrive.expect_tx %9[%c0], %c0 : > } - nvgpu.mbarrier.try_wait.parity %9[%c0], %c0, %c10000000 : > + %phase_c0 = arith.constant 0 : i1 + nvgpu.mbarrier.try_wait.parity %9[%c0], %phase_c0, %c10000000 : > scf.if %10 { %11 = memref.load %7[%c45, %c7] : memref<64x8xf32, 3> %12 = memref.load %8[%c7, %c0] : memref<8x128xf32, 3> -- GitLab From 4588525d7edbc0d14c41f5fa8f3e23a3241a502e Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Mon, 12 Feb 2024 14:05:58 +0900 Subject: [PATCH 032/284] CoverageMappingReader/Writer: MCDCConditionID shouldn't be zero --- llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp | 4 ++++ llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp | 1 + 2 files changed, 5 insertions(+) diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp index fc6014c1f7be..061f0f1bdcd3 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -308,6 +308,10 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray( return Err; if (auto Err = readIntMax(FID, std::numeric_limits::max())) return Err; + if (ID == 0) + return make_error( + coveragemap_error::malformed, + "MCDCConditionID shouldn't be zero"); break; case CounterMappingRegion::MCDCDecisionRegion: Kind = CounterMappingRegion::MCDCDecisionRegion; diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp index 27727f216b05..248a6a79a145 100644 --- a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp +++ b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp @@ -251,6 +251,7 @@ void CoverageMappingWriter::write(raw_ostream &OS) { OS); writeCounter(MinExpressions, Count, OS); writeCounter(MinExpressions, FalseCount, OS); + assert(I->MCDCParams.ID > 0); encodeULEB128(unsigned(I->MCDCParams.ID), OS); encodeULEB128(unsigned(I->MCDCParams.TrueID), OS); encodeULEB128(unsigned(I->MCDCParams.FalseID), OS); -- GitLab From 270f2c5575dc5dd001e91ddc2c71b0f2d23f567c Mon Sep 17 00:00:00 2001 From: martinboehme Date: Tue, 13 Feb 2024 10:01:25 +0100 Subject: [PATCH 033/284] [clang][dataflow] Add `Environment::initializeFieldsWithValues()`. (#81239) This function will be useful when we change the behavior of record-type prvalues so that they directly initialize the associated result object. See also the comment here for more details: https://github.com/llvm/llvm-project/blob/9e73656af524a2c592978aec91de67316c5ce69f/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h#L354 As part of this patch, we document and assert that synthetic fields may not have reference type. There is no practical use case for this: A `StorageLocation` may not have reference type, and a synthetic field of the corresponding non-reference type can serve the same purpose. --- .../FlowSensitive/DataflowAnalysisContext.h | 15 +++- .../FlowSensitive/DataflowEnvironment.h | 8 ++ .../FlowSensitive/DataflowEnvironment.cpp | 74 ++++++++++++------- 3 files changed, 68 insertions(+), 29 deletions(-) diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h index 20e45cc27b01..98bdf037880a 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h @@ -100,6 +100,8 @@ public: /// to add to a `RecordStorageLocation` of a given type. /// Typically, this is called from the constructor of a `DataflowAnalysis` /// + /// The field types returned by the callback may not have reference type. + /// /// To maintain the invariant that all `RecordStorageLocation`s of a given /// type have the same fields: /// * The callback must always return the same result for a given type @@ -205,8 +207,17 @@ public: /// type. llvm::StringMap getSyntheticFields(QualType Type) { assert(Type->isRecordType()); - if (SyntheticFieldCallback) - return SyntheticFieldCallback(Type); + if (SyntheticFieldCallback) { + llvm::StringMap Result = SyntheticFieldCallback(Type); + // Synthetic fields are not allowed to have reference type. + assert([&Result] { + for (const auto &Entry : Result) + if (Entry.getValue()->isReferenceType()) + return false; + return true; + }()); + return Result; + } return {}; } diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index 5c737a561a7c..0aecc749bf41 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -681,6 +681,14 @@ private: llvm::DenseSet &Visited, int Depth, int &CreatedValuesCount); + /// Initializes the fields (including synthetic fields) of `Loc` with values, + /// unless values of the field type are not supported or we hit one of the + /// limits at which we stop producing values (controlled by `Visited`, + /// `Depth`, and `CreatedValuesCount`). + void initializeFieldsWithValues(RecordStorageLocation &Loc, + llvm::DenseSet &Visited, int Depth, + int &CreatedValuesCount); + /// Shared implementation of `createObject()` overloads. /// `D` and `InitExpr` may be null. StorageLocation &createObjectInternal(const ValueDecl *D, QualType Ty, diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index 24811ded970e..93a9dac3bc90 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -887,34 +887,10 @@ Value *Environment::createValueUnlessSelfReferential( if (Type->isRecordType()) { CreatedValuesCount++; - llvm::DenseMap FieldLocs; - for (const FieldDecl *Field : DACtx->getModeledFields(Type)) { - assert(Field != nullptr); + auto &Loc = cast(createStorageLocation(Type)); + initializeFieldsWithValues(Loc, Visited, Depth, CreatedValuesCount); - QualType FieldType = Field->getType(); - - FieldLocs.insert( - {Field, &createLocAndMaybeValue(FieldType, Visited, Depth + 1, - CreatedValuesCount)}); - } - - RecordStorageLocation::SyntheticFieldMap SyntheticFieldLocs; - for (const auto &Entry : DACtx->getSyntheticFields(Type)) { - SyntheticFieldLocs.insert( - {Entry.getKey(), - &createLocAndMaybeValue(Entry.getValue(), Visited, Depth + 1, - CreatedValuesCount)}); - } - - RecordStorageLocation &Loc = DACtx->createRecordStorageLocation( - Type, std::move(FieldLocs), std::move(SyntheticFieldLocs)); - RecordValue &RecordVal = create(Loc); - - // As we already have a storage location for the `RecordValue`, we can and - // should associate them in the environment. - setValue(Loc, RecordVal); - - return &RecordVal; + return &refreshRecordValue(Loc, *this); } return nullptr; @@ -943,6 +919,50 @@ Environment::createLocAndMaybeValue(QualType Ty, return Loc; } +void Environment::initializeFieldsWithValues(RecordStorageLocation &Loc, + llvm::DenseSet &Visited, + int Depth, + int &CreatedValuesCount) { + auto initField = [&](QualType FieldType, StorageLocation &FieldLoc) { + if (FieldType->isRecordType()) { + auto &FieldRecordLoc = cast(FieldLoc); + setValue(FieldRecordLoc, create(FieldRecordLoc)); + initializeFieldsWithValues(FieldRecordLoc, Visited, Depth + 1, + CreatedValuesCount); + } else { + if (!Visited.insert(FieldType.getCanonicalType()).second) + return; + if (Value *Val = createValueUnlessSelfReferential( + FieldType, Visited, Depth + 1, CreatedValuesCount)) + setValue(FieldLoc, *Val); + Visited.erase(FieldType.getCanonicalType()); + } + }; + + for (const auto [Field, FieldLoc] : Loc.children()) { + assert(Field != nullptr); + QualType FieldType = Field->getType(); + + if (FieldType->isReferenceType()) { + Loc.setChild(*Field, + &createLocAndMaybeValue(FieldType, Visited, Depth + 1, + CreatedValuesCount)); + } else { + assert(FieldLoc != nullptr); + initField(FieldType, *FieldLoc); + } + } + for (const auto &[FieldName, FieldLoc] : Loc.synthetic_fields()) { + assert(FieldLoc != nullptr); + QualType FieldType = FieldLoc->getType(); + + // Synthetic fields cannot have reference type, so we don't need to deal + // with this case. + assert(!FieldType->isReferenceType()); + initField(FieldType, Loc.getSyntheticField(FieldName)); + } +} + StorageLocation &Environment::createObjectInternal(const ValueDecl *D, QualType Ty, const Expr *InitExpr) { -- GitLab From 5b015229b77d6ea7916a503d88661b04d4867e7c Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 13 Feb 2024 09:01:45 +0000 Subject: [PATCH 034/284] [AMDGPU] Use LLT::isPointerOrPointerVector in legalizer (#81582) --- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index f3716f96c44a..5458dfc68f3d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -342,11 +342,8 @@ static std::initializer_list AllS64Vectors = {V2S64, V3S64, V4S64, V5S64, // Checks whether a type is in the list of legal register types. static bool isRegisterClassType(LLT Ty) { - if (Ty.isVector() && Ty.getElementType().isPointer()) - Ty = LLT::fixed_vector(Ty.getNumElements(), - LLT::scalar(Ty.getScalarSizeInBits())); - else if (Ty.isPointer()) - Ty = LLT::scalar(Ty.getScalarSizeInBits()); + if (Ty.isPointerOrPointerVector()) + Ty = Ty.changeElementType(LLT::scalar(Ty.getScalarSizeInBits())); return is_contained(AllS32Vectors, Ty) || is_contained(AllS64Vectors, Ty) || is_contained(AllScalarTypes, Ty) || is_contained(AllS16Vectors, Ty); -- GitLab From d860ea96b1d4bcc13bf269e9b108d8f5e0c21d93 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Tue, 13 Feb 2024 09:11:09 +0000 Subject: [PATCH 035/284] [HWASAN] Update dbg.assign intrinsics in HWAsan pass (#79864) llvm.dbg.assign intrinsics have 2 {value, expression} pairs; fix hwasan to update the second expression. Fixes #76545. This is #78606 rebased and with the addition of DPValue handling. Note the addition of --try-experimental-debuginfo-iterators in the tests and some shuffling of code in MemoryTaggingSupport.cpp. --- llvm/lib/IR/DebugInfo.cpp | 4 - .../Instrumentation/HWAddressSanitizer.cpp | 13 ++++ .../Transforms/Utils/MemoryTaggingSupport.cpp | 21 +++-- .../AArch64/dbg-assign-tag-offset-mix-loc.ll | 72 ++++++++++++++++++ .../CodeGen/AArch64/dbg-assign-tag-offset.ll | 76 +++++++++++++++++++ .../declare-to-assign/hwasan.ll | 3 +- .../dbg-assign-tag-offset.ll | 60 +++++++++++++++ 7 files changed, 237 insertions(+), 12 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/dbg-assign-tag-offset-mix-loc.ll create mode 100644 llvm/test/CodeGen/AArch64/dbg-assign-tag-offset.ll create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/dbg-assign-tag-offset.ll diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index d8c1b0d534f6..eaa5cb36a49c 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -2214,10 +2214,6 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) { if (F.hasFnAttribute(Attribute::OptimizeNone)) return /*Changed*/ false; - // FIXME: https://github.com/llvm/llvm-project/issues/76545 - if (F.hasFnAttribute(Attribute::SanitizeHWAddress)) - return /*Changed*/ false; - bool Changed = false; auto *DL = &F.getParent()->getDataLayout(); // Collect a map of {backing storage : dbg.declares} (currently "backing diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index c2a46320b14a..393afc915205 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -1371,6 +1371,14 @@ static bool isLifetimeIntrinsic(Value *V) { return II && II->isLifetimeStartOrEnd(); } +static DbgAssignIntrinsic *DynCastToDbgAssign(DbgVariableIntrinsic *DVI) { + return dyn_cast(DVI); +} + +static DPValue *DynCastToDbgAssign(DPValue *DPV) { + return DPV->isDbgAssign() ? DPV : nullptr; +} + bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo, Value *StackTag, Value *UARTag, const DominatorTree &DT, @@ -1437,6 +1445,11 @@ bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo, if (DPtr->getVariableLocationOp(LocNo) == AI) DPtr->setExpression(DIExpression::appendOpsToArg( DPtr->getExpression(), NewOps, LocNo)); + if (auto *DAI = DynCastToDbgAssign(DPtr)) { + if (DAI->getAddress() == AI) + DAI->setAddressExpression(DIExpression::prependOpcodes( + DAI->getAddressExpression(), NewOps)); + } }; llvm::for_each(Info.DbgVariableIntrinsics, AnnotateDbgRecord); diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp index 43366950d8df..1ffa003db7d5 100644 --- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp +++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp @@ -110,18 +110,22 @@ Instruction *getUntagLocationIfFunctionExit(Instruction &Inst) { } void StackInfoBuilder::visit(Instruction &Inst) { - // Check for non-intrinsic debug-info records. + // Visit non-intrinsic debug-info records attached to Inst. for (auto &DPV : Inst.getDbgValueRange()) { - for (Value *V : DPV.location_ops()) { + auto AddIfInteresting = [&](Value *V) { if (auto *AI = dyn_cast_or_null(V)) { if (!isInterestingAlloca(*AI)) - continue; + return; AllocaInfo &AInfo = Info.AllocasToInstrument[AI]; auto &DPVVec = AInfo.DbgVariableRecords; if (DPVVec.empty() || DPVVec.back() != &DPV) DPVVec.push_back(&DPV); } - } + }; + + for_each(DPV.location_ops(), AddIfInteresting); + if (DPV.isDbgAssign()) + AddIfInteresting(DPV.getAddress()); } if (CallInst *CI = dyn_cast(&Inst)) { @@ -152,16 +156,19 @@ void StackInfoBuilder::visit(Instruction &Inst) { return; } if (auto *DVI = dyn_cast(&Inst)) { - for (Value *V : DVI->location_ops()) { + auto AddIfInteresting = [&](Value *V) { if (auto *AI = dyn_cast_or_null(V)) { if (!isInterestingAlloca(*AI)) - continue; + return; AllocaInfo &AInfo = Info.AllocasToInstrument[AI]; auto &DVIVec = AInfo.DbgVariableIntrinsics; if (DVIVec.empty() || DVIVec.back() != DVI) DVIVec.push_back(DVI); } - } + }; + for_each(DVI->location_ops(), AddIfInteresting); + if (auto *DAI = dyn_cast(DVI)) + AddIfInteresting(DAI->getAddress()); } Instruction *ExitUntag = getUntagLocationIfFunctionExit(Inst); diff --git a/llvm/test/CodeGen/AArch64/dbg-assign-tag-offset-mix-loc.ll b/llvm/test/CodeGen/AArch64/dbg-assign-tag-offset-mix-loc.ll new file mode 100644 index 000000000000..d9d0d982fd9e --- /dev/null +++ b/llvm/test/CodeGen/AArch64/dbg-assign-tag-offset-mix-loc.ll @@ -0,0 +1,72 @@ +; RUN: llc -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s + +;; Similar to dbg-assign-tag-offset.ll except the variable 'x' has been removed +;; and 'y' has an implicit location range as well as stack location range +;; (according to the hand-modified debug info -- see the dbg.value). + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64-unknown-linux-android24" + +; CHECK: DW_TAG_variable +; CHECK-NOT: DW_TAG +; CHECK: DW_AT_LLVM_tag_offset (0x80) +; CHECK-NEXT: DW_AT_name ("y") + +define dso_local void @f() !dbg !14 { + %1 = alloca i32, align 4, !DIAssignID !31 + %2 = alloca i32, align 4, !DIAssignID !32 + call void @llvm.dbg.assign(metadata i1 undef, metadata !20, metadata !DIExpression(), metadata !32, metadata ptr %2, metadata !DIExpression(DW_OP_LLVM_tag_offset, 128)), !dbg !22 + call void @llvm.dbg.value(metadata i32 2, metadata !20, metadata !DIExpression()), !dbg !22 + call void @use(ptr null), !dbg !28 + store i32 1, ptr %2, align 4, !dbg !23, !tbaa !24, !DIAssignID !33 + call void @llvm.dbg.assign(metadata i32 1, metadata !20, metadata !DIExpression(), metadata !33, metadata ptr %2, metadata !DIExpression(DW_OP_LLVM_tag_offset, 128)), !dbg !22 + call void @use(ptr nonnull %1), !dbg !28 + call void @use(ptr nonnull %2), !dbg !29 + ret void, !dbg !30 +} + +declare !dbg !5 void @use(ptr) + +declare void @llvm.dbg.value(metadata, metadata, metadata) +declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!8, !9, !10, !11, !12, !34} +!llvm.ident = !{!13} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None) +!1 = !DIFile(filename: "dbg.cc", directory: "/tmp") +!2 = !{} +!3 = !{!4, !5} +!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64) +!5 = !DISubprogram(name: "use", scope: !1, file: !1, line: 2, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2) +!6 = !DISubroutineType(types: !7) +!7 = !{null, !4} +!8 = !{i32 7, !"Dwarf Version", i32 4} +!9 = !{i32 2, !"Debug Info Version", i32 3} +!10 = !{i32 1, !"wchar_size", i32 4} +!11 = !{i32 7, !"PIC Level", i32 2} +!12 = !{i32 7, !"PIE Level", i32 2} +!13 = !{!"clang version 10.0.0"} +!14 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 4, type: !15, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17) +!15 = !DISubroutineType(types: !16) +!16 = !{null} +!17 = !{!18, !20} +!18 = !DILocalVariable(name: "x", scope: !14, file: !1, line: 5, type: !19) +!19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!20 = !DILocalVariable(name: "y", scope: !14, file: !1, line: 5, type: !19) +!21 = !DILocation(line: 5, column: 3, scope: !14) +!22 = !DILocation(line: 0, scope: !14) +!23 = !DILocation(line: 5, column: 10, scope: !14) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !26, i64 0} +!26 = !{!"omnipotent char", !27, i64 0} +!27 = !{!"Simple C++ TBAA"} +!28 = !DILocation(line: 6, column: 3, scope: !14) +!29 = !DILocation(line: 7, column: 3, scope: !14) +!30 = !DILocation(line: 8, column: 1, scope: !14) +!31 = distinct !DIAssignID() +!32 = distinct !DIAssignID() +!33 = distinct !DIAssignID() +!34 = !{i32 7, !"debug-info-assignment-tracking", i1 true} diff --git a/llvm/test/CodeGen/AArch64/dbg-assign-tag-offset.ll b/llvm/test/CodeGen/AArch64/dbg-assign-tag-offset.ll new file mode 100644 index 000000000000..155e6107158a --- /dev/null +++ b/llvm/test/CodeGen/AArch64/dbg-assign-tag-offset.ll @@ -0,0 +1,76 @@ +; RUN: llc -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s + +;; Copied from dbg-value-tag-offset.ll. Check that variables with locations +;; tracked with dbg.assigns with DW_OP_LLVM_TAG_offset operators in their +;; expressions get a DW_AT_LLVM_tag_offset attribute on their DIE. + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64-unknown-linux-android24" + +; CHECK: DW_TAG_variable +; CHECK-NOT: DW_TAG +; CHECK: DW_AT_LLVM_tag_offset (0x00) +; CHECK-NEXT: DW_AT_name ("x") + +; CHECK: DW_TAG_variable +; CHECK-NOT: DW_TAG +; CHECK: DW_AT_LLVM_tag_offset (0x80) +; CHECK-NEXT: DW_AT_name ("y") + +define dso_local void @f() !dbg !14 { + %1 = alloca i32, align 4, !DIAssignID !31 + call void @llvm.dbg.assign(metadata i1 undef, metadata !18, metadata !DIExpression(), metadata !31, metadata ptr %1, metadata !DIExpression(DW_OP_LLVM_tag_offset, 0)), !dbg !22 + %2 = alloca i32, align 4, !DIAssignID !32 + call void @llvm.dbg.assign(metadata i1 undef, metadata !20, metadata !DIExpression(), metadata !32, metadata ptr %2, metadata !DIExpression(DW_OP_LLVM_tag_offset, 128)), !dbg !22 + store i32 1, ptr %2, align 4, !dbg !23, !tbaa !24, !DIAssignID !33 + call void @llvm.dbg.assign(metadata i32 1, metadata !20, metadata !DIExpression(), metadata !33, metadata ptr %2, metadata !DIExpression(DW_OP_LLVM_tag_offset, 128)), !dbg !22 + call void @use(ptr nonnull %1), !dbg !28 + call void @use(ptr nonnull %2), !dbg !29 + ret void, !dbg !30 +} + +declare !dbg !5 void @use(ptr) + +declare void @llvm.dbg.value(metadata, metadata, metadata) +declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!8, !9, !10, !11, !12, !34} +!llvm.ident = !{!13} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None) +!1 = !DIFile(filename: "dbg.cc", directory: "/tmp") +!2 = !{} +!3 = !{!4, !5} +!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64) +!5 = !DISubprogram(name: "use", scope: !1, file: !1, line: 2, type: !6, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2) +!6 = !DISubroutineType(types: !7) +!7 = !{null, !4} +!8 = !{i32 7, !"Dwarf Version", i32 4} +!9 = !{i32 2, !"Debug Info Version", i32 3} +!10 = !{i32 1, !"wchar_size", i32 4} +!11 = !{i32 7, !"PIC Level", i32 2} +!12 = !{i32 7, !"PIE Level", i32 2} +!13 = !{!"clang version 10.0.0"} +!14 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 4, type: !15, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17) +!15 = !DISubroutineType(types: !16) +!16 = !{null} +!17 = !{!18, !20} +!18 = !DILocalVariable(name: "x", scope: !14, file: !1, line: 5, type: !19) +!19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!20 = !DILocalVariable(name: "y", scope: !14, file: !1, line: 5, type: !19) +!21 = !DILocation(line: 5, column: 3, scope: !14) +!22 = !DILocation(line: 0, scope: !14) +!23 = !DILocation(line: 5, column: 10, scope: !14) +!24 = !{!25, !25, i64 0} +!25 = !{!"int", !26, i64 0} +!26 = !{!"omnipotent char", !27, i64 0} +!27 = !{!"Simple C++ TBAA"} +!28 = !DILocation(line: 6, column: 3, scope: !14) +!29 = !DILocation(line: 7, column: 3, scope: !14) +!30 = !DILocation(line: 8, column: 1, scope: !14) +!31 = distinct !DIAssignID() +!32 = distinct !DIAssignID() +!33 = distinct !DIAssignID() +!34 = !{i32 7, !"debug-info-assignment-tracking", i1 true} diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/declare-to-assign/hwasan.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/declare-to-assign/hwasan.ll index c4b209de7701..f7f126cf3c51 100644 --- a/llvm/test/DebugInfo/Generic/assignment-tracking/declare-to-assign/hwasan.ll +++ b/llvm/test/DebugInfo/Generic/assignment-tracking/declare-to-assign/hwasan.ll @@ -1,6 +1,7 @@ ; RUN: opt %s -S -passes=declare-to-assign -o - | FileCheck %s +; RUN: opt --try-experimental-debuginfo-iterators %s -S -passes=declare-to-assign -o - | FileCheck %s -; CHECK: call void @llvm.dbg.declare +; CHECK: call void @llvm.dbg.assign define dso_local void @f() sanitize_hwaddress !dbg !9 { entry: diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/dbg-assign-tag-offset.ll b/llvm/test/Instrumentation/HWAddressSanitizer/dbg-assign-tag-offset.ll new file mode 100644 index 000000000000..ec8d0340de45 --- /dev/null +++ b/llvm/test/Instrumentation/HWAddressSanitizer/dbg-assign-tag-offset.ll @@ -0,0 +1,60 @@ +; RUN: opt -passes=hwasan -S -o - %s | FileCheck %s +; RUN: opt --try-experimental-debuginfo-iterators -passes=hwasan -S -o - %s | FileCheck %s + +source_filename = "test.ll" +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64--linux-android" + +declare void @g(ptr, ptr, ptr, ptr, ptr, ptr) + +; Function Attrs: sanitize_hwaddress +define void @f() #0 !dbg !7 { +entry: + %nodebug0 = alloca ptr, align 8 + %nodebug1 = alloca ptr, align 8 + %nodebug2 = alloca ptr, align 8 + %nodebug3 = alloca ptr, align 8 + ; CHECK: %a = alloca{{.*}} !DIAssignID ![[ID1:[0-9]+]] + %a = alloca ptr, align 8, !DIAssignID !13 + ; CHECK: @llvm.dbg.assign{{.*}} metadata ![[ID1]]{{.*}} !DIExpression(DW_OP_LLVM_tag_offset, 32) + call void @llvm.dbg.assign(metadata i1 undef, metadata !14, metadata !DIExpression(), metadata !13, metadata ptr %a, metadata !DIExpression()), !dbg !15 + ; CHECK: %b = alloca{{.*}} !DIAssignID ![[ID2:[0-9]+]] + %b = alloca ptr, align 8, !DIAssignID !16 + ; CHECK: @llvm.dbg.assign{{.*}} metadata ![[ID2]]{{.*}} !DIExpression(DW_OP_LLVM_tag_offset, 96) + call void @llvm.dbg.assign(metadata i1 undef, metadata !17, metadata !DIExpression(), metadata !16, metadata ptr %b, metadata !DIExpression()), !dbg !15 + call void @g(ptr %nodebug0, ptr %nodebug1, ptr %nodebug2, ptr %nodebug3, ptr %a, ptr %b) + ret void, !dbg !18 +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) #1 + +attributes #0 = { sanitize_hwaddress } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) +!1 = !DIFile(filename: "x.c", directory: "/") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 7, !"debug-info-assignment-tracking", i1 true} +!6 = !{!"clang"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64) +!11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !12) +!12 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char) +!13 = distinct !DIAssignID() +!14 = !DILocalVariable(name: "a", scope: !7, file: !1, line: 1, type: !10) +!15 = !DILocation(line: 0, scope: !7) +!16 = distinct !DIAssignID() +!17 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 1, type: !10) +!18 = !DILocation(line: 1, column: 37, scope: !7) -- GitLab From 44706bd4f0e26f86eda24e4888044897fd4f92a8 Mon Sep 17 00:00:00 2001 From: ostannard Date: Tue, 13 Feb 2024 09:13:22 +0000 Subject: [PATCH 036/284] [InstCombine] Don't add fcmp instructions to strictfp functions (#81498) The strictfp attribute has the requirement that "LLVM will not introduce any new floating-point instructions that may trap". The llvm.is.fpclass intrinsic is documented as "The function never raises floating-point exceptions", and the fcmp instruction may raise one, so we can't transform the former into the latter in functions with the strictfp attribute. --- .../InstCombine/InstCombineCalls.cpp | 3 +- .../InstCombine/fpclass-check-idioms.ll | 323 +++++++++++++++++- 2 files changed, 316 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 56d1259e9551..5266808c5aba 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -902,7 +902,8 @@ Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) { const FPClassTest OrderedMask = Mask & ~fcNan; const FPClassTest OrderedInvertedMask = ~OrderedMask & ~fcNan; - const bool IsStrict = II.isStrictFP(); + const bool IsStrict = + II.getFunction()->getAttributes().hasFnAttr(Attribute::StrictFP); Value *FNegSrc; if (match(Src0, m_FNeg(m_Value(FNegSrc)))) { diff --git a/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll b/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll index d2b4536448eb..42c6506e34cf 100644 --- a/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll +++ b/llvm/test/Transforms/InstCombine/fpclass-check-idioms.ll @@ -14,6 +14,18 @@ define i1 @f32_fcnan_fcinf(float %a) { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 519) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2139095040 + %cmp = icmp eq i32 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_not_fcnan_fcinf(float %a) { ; CHECK-LABEL: define i1 @f32_not_fcnan_fcinf( ; CHECK-SAME: float [[A:%.*]]) { @@ -27,6 +39,18 @@ define i1 @f32_not_fcnan_fcinf(float %a) { ret i1 %cmp } +define i1 @f32_not_fcnan_fcinf_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_not_fcnan_fcinf_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 504) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2139095040 + %cmp = icmp ne i32 %and, 2139095040 + ret i1 %cmp +} + define i1 @f64_fcnan_fcinf(double %a) { ; CHECK-LABEL: define i1 @f64_fcnan_fcinf( ; CHECK-SAME: double [[A:%.*]]) { @@ -40,6 +64,18 @@ define i1 @f64_fcnan_fcinf(double %a) { ret i1 %cmp } +define i1 @f64_fcnan_fcinf_strictfp(double %a) strictfp { +; CHECK-LABEL: define i1 @f64_fcnan_fcinf_strictfp( +; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f64(double [[A]], i32 519) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i64 = bitcast double %a to i64 + %and = and i64 %i64, 9218868437227405312 + %cmp = icmp eq i64 %and, 9218868437227405312 + ret i1 %cmp +} + define i1 @f32_fcinf(float %a) { ; CHECK-LABEL: define i1 @f32_fcinf( ; CHECK-SAME: float [[A:%.*]]) { @@ -53,6 +89,18 @@ define i1 @f32_fcinf(float %a) { ret i1 %cmp } +define i1 @f32_fcinf_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcinf_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 516) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2147483647 + %cmp = icmp eq i32 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcposinf(float %a) { ; CHECK-LABEL: define i1 @f32_fcposinf( ; CHECK-SAME: float [[A:%.*]]) { @@ -64,6 +112,17 @@ define i1 @f32_fcposinf(float %a) { ret i1 %cmp } +define i1 @f32_fcposinf_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposinf_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 512) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp eq i32 %i32, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcneginf(float %a) { ; CHECK-LABEL: define i1 @f32_fcneginf( ; CHECK-SAME: float [[A:%.*]]) { @@ -75,6 +134,17 @@ define i1 @f32_fcneginf(float %a) { ret i1 %cmp } +define i1 @f32_fcneginf_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcneginf_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 4) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp eq i32 %i32, 4286578688 + ret i1 %cmp +} + define i1 @f32_fcposzero(float %a) { ; CHECK-LABEL: define i1 @f32_fcposzero( ; CHECK-SAME: float [[A:%.*]]) { @@ -86,6 +156,17 @@ define i1 @f32_fcposzero(float %a) { ret i1 %cmp } +define i1 @f32_fcposzero_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposzero_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 64) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp eq i32 %i32, 0 + ret i1 %cmp +} + define i1 @f32_fcnegzero(float %a) { ; CHECK-LABEL: define i1 @f32_fcnegzero( ; CHECK-SAME: float [[A:%.*]]) { @@ -97,6 +178,17 @@ define i1 @f32_fcnegzero(float %a) { ret i1 %cmp } +define i1 @f32_fcnegzero_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnegzero_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 32) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp eq i32 %i32, 2147483648 + ret i1 %cmp +} + define i1 @f32_fczero(float %a) { ; CHECK-LABEL: define i1 @f32_fczero( ; CHECK-SAME: float [[A:%.*]]) { @@ -109,6 +201,18 @@ define i1 @f32_fczero(float %a) { ret i1 %cmp } +define i1 @f32_fczero_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fczero_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 96) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2147483647 + %cmp = icmp eq i32 %and, 0 + ret i1 %cmp +} + ; TODO: handle more fpclass check idioms define i1 @f32_fcnan(float %a) { ; CHECK-LABEL: define i1 @f32_fcnan( @@ -130,17 +234,24 @@ define i1 @f32_fcnan(float %a) { ret i1 %res } -define i1 @f32_fcnan_fcinf_strictfp(float %a) strictfp { -; CHECK-LABEL: define i1 @f32_fcnan_fcinf_strictfp( -; CHECK-SAME: float [[A:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[A]]) -; CHECK-NEXT: [[CMP:%.*]] = fcmp ueq float [[TMP1]], 0x7FF0000000000000 -; CHECK-NEXT: ret i1 [[CMP]] +define i1 @f32_fcnan_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[AND1:%.*]] = and i32 [[I32]], 2139095040 +; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i32 [[AND1]], 2139095040 +; CHECK-NEXT: [[AND2:%.*]] = and i32 [[I32]], 8388607 +; CHECK-NEXT: [[CMP2:%.*]] = icmp ne i32 [[AND2]], 0 +; CHECK-NEXT: [[RES:%.*]] = and i1 [[CMP1]], [[CMP2]] +; CHECK-NEXT: ret i1 [[RES]] ; %i32 = bitcast float %a to i32 - %and = and i32 %i32, 2139095040 - %cmp = icmp eq i32 %and, 2139095040 - ret i1 %cmp + %and1 = and i32 %i32, 2139095040 + %cmp1 = icmp eq i32 %and1, 2139095040 + %and2 = and i32 %i32, 8388607 + %cmp2 = icmp ne i32 %and2, 0 + %res = and i1 %cmp1, %cmp2 + ret i1 %res } define <2 x i1> @f32_fcnan_fcinf_vec(<2 x float> %a) { @@ -156,6 +267,18 @@ define <2 x i1> @f32_fcnan_fcinf_vec(<2 x float> %a) { ret <2 x i1> %cmp } +define <2 x i1> @f32_fcnan_fcinf_vec_strictfp(<2 x float> %a) strictfp { +; CHECK-LABEL: define <2 x i1> @f32_fcnan_fcinf_vec_strictfp( +; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call <2 x i1> @llvm.is.fpclass.v2f32(<2 x float> [[A]], i32 519) +; CHECK-NEXT: ret <2 x i1> [[CMP]] +; + %i32 = bitcast <2 x float> %a to <2 x i32> + %and = and <2 x i32> %i32, + %cmp = icmp eq <2 x i32> %and, + ret <2 x i1> %cmp +} + define <2 x i1> @f32_fcinf_vec(<2 x float> %a) { ; CHECK-LABEL: define <2 x i1> @f32_fcinf_vec( ; CHECK-SAME: <2 x float> [[A:%.*]]) { @@ -169,6 +292,18 @@ define <2 x i1> @f32_fcinf_vec(<2 x float> %a) { ret <2 x i1> %cmp } +define <2 x i1> @f32_fcinf_vec_strictfp(<2 x float> %a) strictfp { +; CHECK-LABEL: define <2 x i1> @f32_fcinf_vec_strictfp( +; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call <2 x i1> @llvm.is.fpclass.v2f32(<2 x float> [[A]], i32 516) +; CHECK-NEXT: ret <2 x i1> [[CMP]] +; + %i32 = bitcast <2 x float> %a to <2 x i32> + %and = and <2 x i32> %i32, + %cmp = icmp eq <2 x i32> %and, + ret <2 x i1> %cmp +} + ; Negative tests define i1 @f32_fcnan_fcinf_wrong_mask1(float %a) { @@ -185,6 +320,20 @@ define i1 @f32_fcnan_fcinf_wrong_mask1(float %a) { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_wrong_mask1_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_mask1_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[AND:%.*]] = and i32 [[I32]], 2139095041 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[AND]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2139095041 + %cmp = icmp eq i32 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcnan_fcinf_wrong_mask2(float %a) { ; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_mask2( ; CHECK-SAME: float [[A:%.*]]) { @@ -199,6 +348,20 @@ define i1 @f32_fcnan_fcinf_wrong_mask2(float %a) { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_wrong_mask2_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_mask2_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[AND:%.*]] = and i32 [[I32]], 2139095040 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[AND]], 2130706432 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2139095040 + %cmp = icmp eq i32 %and, 2130706432 + ret i1 %cmp +} + define i1 @f64_fcnan_fcinf_wrong_mask3(double %a) { ; CHECK-LABEL: define i1 @f64_fcnan_fcinf_wrong_mask3( ; CHECK-SAME: double [[A:%.*]]) { @@ -213,6 +376,20 @@ define i1 @f64_fcnan_fcinf_wrong_mask3(double %a) { ret i1 %cmp } +define i1 @f64_fcnan_fcinf_wrong_mask3_strictfp(double %a) strictfp { +; CHECK-LABEL: define i1 @f64_fcnan_fcinf_wrong_mask3_strictfp( +; CHECK-SAME: double [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I64:%.*]] = bitcast double [[A]] to i64 +; CHECK-NEXT: [[AND:%.*]] = and i64 [[I64]], 2139095040 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[AND]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i64 = bitcast double %a to i64 + %and = and i64 %i64, 2139095040 + %cmp = icmp eq i64 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcnan_fcinf_wrong_pred(float %a) { ; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_pred( ; CHECK-SAME: float [[A:%.*]]) { @@ -226,6 +403,18 @@ define i1 @f32_fcnan_fcinf_wrong_pred(float %a) { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_wrong_pred_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_pred_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[A]], i32 504) +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2139095040 + %cmp = icmp slt i32 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcposzero_wrong_pred(float %a) { ; CHECK-LABEL: define i1 @f32_fcposzero_wrong_pred( ; CHECK-SAME: float [[A:%.*]]) { @@ -238,6 +427,18 @@ define i1 @f32_fcposzero_wrong_pred(float %a) { ret i1 %cmp } +define i1 @f32_fcposzero_wrong_pred_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposzero_wrong_pred_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I32]], 0 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp slt i32 %i32, 0 + ret i1 %cmp +} + define i1 @f32_fcnan_fcinf_wrong_type1(<2 x float> %a) { ; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_type1( ; CHECK-SAME: <2 x float> [[A:%.*]]) { @@ -252,6 +453,20 @@ define i1 @f32_fcnan_fcinf_wrong_type1(<2 x float> %a) { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_wrong_type1_strictfp(<2 x float> %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_type1_strictfp( +; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I64:%.*]] = bitcast <2 x float> [[A]] to i64 +; CHECK-NEXT: [[AND:%.*]] = and i64 [[I64]], 2139095040 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[AND]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i64 = bitcast <2 x float> %a to i64 + %and = and i64 %i64, 2139095040 + %cmp = icmp eq i64 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcposinf_wrong_type1(<2 x float> %a) { ; CHECK-LABEL: define i1 @f32_fcposinf_wrong_type1( ; CHECK-SAME: <2 x float> [[A:%.*]]) { @@ -264,6 +479,18 @@ define i1 @f32_fcposinf_wrong_type1(<2 x float> %a) { ret i1 %cmp } +define i1 @f32_fcposinf_wrong_type1_strictfp(<2 x float> %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposinf_wrong_type1_strictfp( +; CHECK-SAME: <2 x float> [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I64:%.*]] = bitcast <2 x float> [[A]] to i64 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[I64]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i64 = bitcast <2 x float> %a to i64 + %cmp = icmp eq i64 %i64, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcnan_fcinf_wrong_type2(x86_fp80 %a) { ; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_type2( ; CHECK-SAME: x86_fp80 [[A:%.*]]) { @@ -278,6 +505,20 @@ define i1 @f32_fcnan_fcinf_wrong_type2(x86_fp80 %a) { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_wrong_type2_strictfp(x86_fp80 %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_wrong_type2_strictfp( +; CHECK-SAME: x86_fp80 [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I80:%.*]] = bitcast x86_fp80 [[A]] to i80 +; CHECK-NEXT: [[AND:%.*]] = and i80 [[I80]], 2139095040 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i80 [[AND]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i80 = bitcast x86_fp80 %a to i80 + %and = and i80 %i80, 2139095040 + %cmp = icmp eq i80 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcposzero_wrong_type2(x86_fp80 %a) { ; CHECK-LABEL: define i1 @f32_fcposzero_wrong_type2( ; CHECK-SAME: x86_fp80 [[A:%.*]]) { @@ -290,6 +531,18 @@ define i1 @f32_fcposzero_wrong_type2(x86_fp80 %a) { ret i1 %cmp } +define i1 @f32_fcposzero_wrong_type2_strictfp(x86_fp80 %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposzero_wrong_type2_strictfp( +; CHECK-SAME: x86_fp80 [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I80:%.*]] = bitcast x86_fp80 [[A]] to i80 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i80 [[I80]], 0 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i80 = bitcast x86_fp80 %a to i80 + %cmp = icmp eq i80 %i80, 0 + ret i1 %cmp +} + define i1 @f32_fcnan_fcinf_noimplicitfloat(float %a) #0 { ; CHECK-LABEL: define i1 @f32_fcnan_fcinf_noimplicitfloat( ; CHECK-SAME: float [[A:%.*]]) #[[ATTR1:[0-9]+]] { @@ -304,6 +557,20 @@ define i1 @f32_fcnan_fcinf_noimplicitfloat(float %a) #0 { ret i1 %cmp } +define i1 @f32_fcnan_fcinf_noimplicitfloat_strictfp(float %a) strictfp #0 { +; CHECK-LABEL: define i1 @f32_fcnan_fcinf_noimplicitfloat_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[AND:%.*]] = and i32 [[I32]], 2139095040 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[AND]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %and = and i32 %i32, 2139095040 + %cmp = icmp eq i32 %and, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcposinf_noimplicitfloat(float %a) #0 { ; CHECK-LABEL: define i1 @f32_fcposinf_noimplicitfloat( ; CHECK-SAME: float [[A:%.*]]) #[[ATTR1]] { @@ -316,6 +583,18 @@ define i1 @f32_fcposinf_noimplicitfloat(float %a) #0 { ret i1 %cmp } +define i1 @f32_fcposinf_noimplicitfloat_strictfp(float %a) strictfp #0 { +; CHECK-LABEL: define i1 @f32_fcposinf_noimplicitfloat_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[I32]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp eq i32 %i32, 2139095040 + ret i1 %cmp +} + define i1 @f32_fcposnan(float %a) { ; CHECK-LABEL: define i1 @f32_fcposnan( ; CHECK-SAME: float [[A:%.*]]) { @@ -328,6 +607,18 @@ define i1 @f32_fcposnan(float %a) { ret i1 %cmp } +define i1 @f32_fcposnan_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposnan_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[I32]], 2139095041 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + %cmp = icmp eq i32 %i32, 2139095041 + ret i1 %cmp +} + define i1 @f32_fcposinf_multiuse(float %a) { ; CHECK-LABEL: define i1 @f32_fcposinf_multiuse( ; CHECK-SAME: float [[A:%.*]]) { @@ -342,6 +633,20 @@ define i1 @f32_fcposinf_multiuse(float %a) { ret i1 %cmp } +define i1 @f32_fcposinf_multiuse_strictfp(float %a) strictfp { +; CHECK-LABEL: define i1 @f32_fcposinf_multiuse_strictfp( +; CHECK-SAME: float [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[I32:%.*]] = bitcast float [[A]] to i32 +; CHECK-NEXT: call void @usei32(i32 [[I32]]) +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[I32]], 2139095040 +; CHECK-NEXT: ret i1 [[CMP]] +; + %i32 = bitcast float %a to i32 + call void @usei32(i32 %i32) + %cmp = icmp eq i32 %i32, 2139095040 + ret i1 %cmp +} + declare void @usei32(i32) attributes #0 = { noimplicitfloat } -- GitLab From ca61e6a71dbe622593b27ab6c6f3bfd058069772 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Tue, 13 Feb 2024 17:28:06 +0800 Subject: [PATCH 037/284] Revert "[CVP] Check whether the default case is reachable (#79993)" (#81585) This reverts commit a034e65e972175a2465deacb8c78bc7efc99bd23. Some protobuf users reported that this patch caused a significant compile-time regression because `TailDuplicator` works poorly with a specific pattern. We will reland it once the codegen issue is fixed. --- .../Scalar/CorrelatedValuePropagation.cpp | 34 -- .../CorrelatedValuePropagation/basic.ll | 11 +- .../CorrelatedValuePropagation/switch.ll | 301 ------------------ 3 files changed, 5 insertions(+), 341 deletions(-) delete mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/switch.ll diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index 24e1677bf643..9235850de92f 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -371,7 +371,6 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI, { // Scope for SwitchInstProfUpdateWrapper. It must not live during // ConstantFoldTerminator() as the underlying SwitchInst can be changed. SwitchInstProfUpdateWrapper SI(*I); - unsigned ReachableCaseCount = 0; for (auto CI = SI->case_begin(), CE = SI->case_end(); CI != CE;) { ConstantInt *Case = CI->getCaseValue(); @@ -408,33 +407,6 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI, // Increment the case iterator since we didn't delete it. ++CI; - ++ReachableCaseCount; - } - - BasicBlock *DefaultDest = SI->getDefaultDest(); - if (ReachableCaseCount > 1 && - !isa(DefaultDest->getFirstNonPHIOrDbg())) { - ConstantRange CR = LVI->getConstantRangeAtUse(I->getOperandUse(0), - /*UndefAllowed*/ false); - // The default dest is unreachable if all cases are covered. - if (!CR.isSizeLargerThan(ReachableCaseCount)) { - BasicBlock *NewUnreachableBB = - BasicBlock::Create(BB->getContext(), "default.unreachable", - BB->getParent(), DefaultDest); - new UnreachableInst(BB->getContext(), NewUnreachableBB); - - DefaultDest->removePredecessor(BB); - SI->setDefaultDest(NewUnreachableBB); - - if (SuccessorsCount[DefaultDest] == 1) - DTU.applyUpdatesPermissive( - {{DominatorTree::Delete, BB, DefaultDest}}); - DTU.applyUpdatesPermissive( - {{DominatorTree::Insert, BB, NewUnreachableBB}}); - - ++NumDeadCases; - Changed = true; - } } } @@ -1255,12 +1227,6 @@ CorrelatedValuePropagationPass::run(Function &F, FunctionAnalysisManager &AM) { if (!Changed) { PA = PreservedAnalyses::all(); } else { -#if defined(EXPENSIVE_CHECKS) - assert(DT->verify(DominatorTree::VerificationLevel::Full)); -#else - assert(DT->verify(DominatorTree::VerificationLevel::Fast)); -#endif // EXPENSIVE_CHECKS - PA.preserve(); PA.preserve(); } diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll index 93ed008022e2..8dce9ef9fa43 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll @@ -442,7 +442,7 @@ define i32 @switch_range(i32 %cond) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[S:%.*]] = urem i32 [[COND:%.*]], 3 ; CHECK-NEXT: [[S1:%.*]] = add nuw nsw i32 [[S]], 1 -; CHECK-NEXT: switch i32 [[S1]], label [[DEFAULT_UNREACHABLE:%.*]] [ +; CHECK-NEXT: switch i32 [[S1]], label [[UNREACHABLE:%.*]] [ ; CHECK-NEXT: i32 1, label [[EXIT1:%.*]] ; CHECK-NEXT: i32 2, label [[EXIT2:%.*]] ; CHECK-NEXT: i32 3, label [[EXIT1]] @@ -451,8 +451,6 @@ define i32 @switch_range(i32 %cond) { ; CHECK-NEXT: ret i32 1 ; CHECK: exit2: ; CHECK-NEXT: ret i32 2 -; CHECK: default.unreachable: -; CHECK-NEXT: unreachable ; CHECK: unreachable: ; CHECK-NEXT: ret i32 0 ; @@ -515,9 +513,10 @@ define i8 @switch_defaultdest_multipleuse(i8 %t0) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[O:%.*]] = or i8 [[T0:%.*]], 1 ; CHECK-NEXT: [[R:%.*]] = srem i8 1, [[O]] -; CHECK-NEXT: br label [[EXIT:%.*]] -; CHECK: default.unreachable: -; CHECK-NEXT: unreachable +; CHECK-NEXT: switch i8 [[R]], label [[EXIT:%.*]] [ +; CHECK-NEXT: i8 0, label [[EXIT]] +; CHECK-NEXT: i8 1, label [[EXIT]] +; CHECK-NEXT: ] ; CHECK: exit: ; CHECK-NEXT: ret i8 0 ; diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll b/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll deleted file mode 100644 index a0794d5efe93..000000000000 --- a/llvm/test/Transforms/CorrelatedValuePropagation/switch.ll +++ /dev/null @@ -1,301 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s - -define i32 @test_unreachable_default(i32 noundef %num) { -; CHECK-LABEL: define i32 @test_unreachable_default( -; CHECK-SAME: i32 noundef [[NUM:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[SUB:%.*]] = add i32 [[NUM]], -120 -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[SUB]], 3 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 2 -; CHECK-NEXT: switch i32 [[COND]], label [[DEFAULT_UNREACHABLE:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB2:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: sw.bb: -; CHECK-NEXT: [[CALL:%.*]] = call i32 @call0() -; CHECK-NEXT: br label [[CLEANUP:%.*]] -; CHECK: sw.bb2: -; CHECK-NEXT: [[CALL3:%.*]] = call i32 @call1() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.bb4: -; CHECK-NEXT: [[CALL5:%.*]] = call i32 @call2() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: default.unreachable: -; CHECK-NEXT: unreachable -; CHECK: sw.default: -; CHECK-NEXT: [[CALL6:%.*]] = call i32 @call3() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: cleanup: -; CHECK-NEXT: [[RETVAL_0:%.*]] = phi i32 [ [[CALL6]], [[SW_DEFAULT:%.*]] ], [ [[CALL5]], [[SW_BB4]] ], [ [[CALL3]], [[SW_BB2]] ], [ [[CALL]], [[SW_BB]] ] -; CHECK-NEXT: ret i32 [[RETVAL_0]] -; -entry: - %sub = add i32 %num, -120 - %cmp = icmp ult i32 %sub, 3 - %cond = select i1 %cmp, i32 %sub, i32 2 - switch i32 %cond, label %sw.default [ - i32 0, label %sw.bb - i32 1, label %sw.bb2 - i32 2, label %sw.bb4 - ] - -sw.bb: - %call = call i32 @call0() - br label %cleanup - -sw.bb2: - %call3 = call i32 @call1() - br label %cleanup - -sw.bb4: - %call5 = call i32 @call2() - br label %cleanup - -sw.default: - %call6 = call i32 @call3() - br label %cleanup - -cleanup: - %retval.0 = phi i32 [ %call6, %sw.default ], [ %call5, %sw.bb4 ], [ %call3, %sw.bb2 ], [ %call, %sw.bb ] - ret i32 %retval.0 -} - -define i32 @test_unreachable_default_shared_edge(i32 noundef %num) { -; CHECK-LABEL: define i32 @test_unreachable_default_shared_edge( -; CHECK-SAME: i32 noundef [[NUM:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[SUB:%.*]] = add i32 [[NUM]], -120 -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[SUB]], 3 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 2 -; CHECK-NEXT: switch i32 [[COND]], label [[DEFAULT_UNREACHABLE:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB2:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: sw.bb: -; CHECK-NEXT: [[CALL:%.*]] = call i32 @call0() -; CHECK-NEXT: br label [[CLEANUP:%.*]] -; CHECK: sw.bb2: -; CHECK-NEXT: [[CALL3:%.*]] = call i32 @call1() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: default.unreachable: -; CHECK-NEXT: unreachable -; CHECK: sw.bb4: -; CHECK-NEXT: [[CALL5:%.*]] = call i32 @call4(i32 [[SUB]]) -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: cleanup: -; CHECK-NEXT: [[RETVAL_0:%.*]] = phi i32 [ [[CALL5]], [[SW_BB4]] ], [ [[CALL3]], [[SW_BB2]] ], [ [[CALL]], [[SW_BB]] ] -; CHECK-NEXT: ret i32 [[RETVAL_0]] -; -entry: - %sub = add i32 %num, -120 - %cmp = icmp ult i32 %sub, 3 - %cond = select i1 %cmp, i32 %sub, i32 2 - switch i32 %cond, label %sw.bb4 [ - i32 0, label %sw.bb - i32 1, label %sw.bb2 - i32 2, label %sw.bb4 - ] - -sw.bb: - %call = call i32 @call0() - br label %cleanup - -sw.bb2: - %call3 = call i32 @call1() - br label %cleanup - -sw.bb4: - %val = phi i32 [ %sub, %entry ], [ %sub, %entry ] - %call5 = call i32 @call4(i32 %val) - br label %cleanup - -cleanup: - %retval.0 = phi i32 [ %call5, %sw.bb4 ], [ %call3, %sw.bb2 ], [ %call, %sw.bb ] - ret i32 %retval.0 -} - -; Negative tests - -define i32 @test_reachable_default(i32 noundef %num) { -; CHECK-LABEL: define i32 @test_reachable_default( -; CHECK-SAME: i32 noundef [[NUM:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[SUB:%.*]] = add i32 [[NUM]], -120 -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[SUB]], 3 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 4 -; CHECK-NEXT: switch i32 [[COND]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB2:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: sw.bb: -; CHECK-NEXT: [[CALL:%.*]] = call i32 @call0() -; CHECK-NEXT: br label [[CLEANUP:%.*]] -; CHECK: sw.bb2: -; CHECK-NEXT: [[CALL3:%.*]] = call i32 @call1() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.bb4: -; CHECK-NEXT: [[CALL5:%.*]] = call i32 @call2() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.default: -; CHECK-NEXT: [[CALL6:%.*]] = call i32 @call3() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: cleanup: -; CHECK-NEXT: [[RETVAL_0:%.*]] = phi i32 [ [[CALL6]], [[SW_DEFAULT]] ], [ [[CALL5]], [[SW_BB4]] ], [ [[CALL3]], [[SW_BB2]] ], [ [[CALL]], [[SW_BB]] ] -; CHECK-NEXT: ret i32 [[RETVAL_0]] -; -entry: - %sub = add i32 %num, -120 - %cmp = icmp ult i32 %sub, 3 - %cond = select i1 %cmp, i32 %sub, i32 4 - switch i32 %cond, label %sw.default [ - i32 0, label %sw.bb - i32 1, label %sw.bb2 - i32 2, label %sw.bb4 - ] - -sw.bb: - %call = call i32 @call0() - br label %cleanup - -sw.bb2: - %call3 = call i32 @call1() - br label %cleanup - -sw.bb4: - %call5 = call i32 @call2() - br label %cleanup - -sw.default: - %call6 = call i32 @call3() - br label %cleanup - -cleanup: - %retval.0 = phi i32 [ %call6, %sw.default ], [ %call5, %sw.bb4 ], [ %call3, %sw.bb2 ], [ %call, %sw.bb ] - ret i32 %retval.0 -} - -define i32 @test_unreachable_default_cond_may_be_undef(i32 %num) { -; CHECK-LABEL: define i32 @test_unreachable_default_cond_may_be_undef( -; CHECK-SAME: i32 [[NUM:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[SUB:%.*]] = add i32 [[NUM]], -120 -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[SUB]], 3 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 2 -; CHECK-NEXT: switch i32 [[COND]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB2:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: sw.bb: -; CHECK-NEXT: [[CALL:%.*]] = call i32 @call0() -; CHECK-NEXT: br label [[CLEANUP:%.*]] -; CHECK: sw.bb2: -; CHECK-NEXT: [[CALL3:%.*]] = call i32 @call1() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.bb4: -; CHECK-NEXT: [[CALL5:%.*]] = call i32 @call2() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.default: -; CHECK-NEXT: [[CALL6:%.*]] = call i32 @call3() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: cleanup: -; CHECK-NEXT: [[RETVAL_0:%.*]] = phi i32 [ [[CALL6]], [[SW_DEFAULT]] ], [ [[CALL5]], [[SW_BB4]] ], [ [[CALL3]], [[SW_BB2]] ], [ [[CALL]], [[SW_BB]] ] -; CHECK-NEXT: ret i32 [[RETVAL_0]] -; -entry: - %sub = add i32 %num, -120 - %cmp = icmp ult i32 %sub, 3 - %cond = select i1 %cmp, i32 %sub, i32 2 - switch i32 %cond, label %sw.default [ - i32 0, label %sw.bb - i32 1, label %sw.bb2 - i32 2, label %sw.bb4 - ] - -sw.bb: - %call = call i32 @call0() - br label %cleanup - -sw.bb2: - %call3 = call i32 @call1() - br label %cleanup - -sw.bb4: - %call5 = call i32 @call2() - br label %cleanup - -sw.default: - %call6 = call i32 @call3() - br label %cleanup - -cleanup: - %retval.0 = phi i32 [ %call6, %sw.default ], [ %call5, %sw.bb4 ], [ %call3, %sw.bb2 ], [ %call, %sw.bb ] - ret i32 %retval.0 -} - -define i32 @test_default_is_already_unreachable(i32 %num) { -; CHECK-LABEL: define i32 @test_default_is_already_unreachable( -; CHECK-SAME: i32 [[NUM:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[SUB:%.*]] = add i32 [[NUM]], -120 -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[SUB]], 3 -; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 2 -; CHECK-NEXT: switch i32 [[COND]], label [[SW_DEFAULT:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] -; CHECK-NEXT: i32 1, label [[SW_BB2:%.*]] -; CHECK-NEXT: i32 2, label [[SW_BB4:%.*]] -; CHECK-NEXT: ] -; CHECK: sw.bb: -; CHECK-NEXT: [[CALL:%.*]] = call i32 @call0() -; CHECK-NEXT: br label [[CLEANUP:%.*]] -; CHECK: sw.bb2: -; CHECK-NEXT: [[CALL3:%.*]] = call i32 @call1() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.bb4: -; CHECK-NEXT: [[CALL5:%.*]] = call i32 @call2() -; CHECK-NEXT: br label [[CLEANUP]] -; CHECK: sw.default: -; CHECK-NEXT: unreachable -; CHECK: cleanup: -; CHECK-NEXT: [[RETVAL_0:%.*]] = phi i32 [ [[CALL5]], [[SW_BB4]] ], [ [[CALL3]], [[SW_BB2]] ], [ [[CALL]], [[SW_BB]] ] -; CHECK-NEXT: ret i32 [[RETVAL_0]] -; -entry: - %sub = add i32 %num, -120 - %cmp = icmp ult i32 %sub, 3 - %cond = select i1 %cmp, i32 %sub, i32 2 - switch i32 %cond, label %sw.default [ - i32 0, label %sw.bb - i32 1, label %sw.bb2 - i32 2, label %sw.bb4 - ] - -sw.bb: - %call = call i32 @call0() - br label %cleanup - -sw.bb2: - %call3 = call i32 @call1() - br label %cleanup - -sw.bb4: - %call5 = call i32 @call2() - br label %cleanup - -sw.default: - unreachable - -cleanup: - %retval.0 = phi i32 [ %call5, %sw.bb4 ], [ %call3, %sw.bb2 ], [ %call, %sw.bb ] - ret i32 %retval.0 -} - -declare i32 @call0() -declare i32 @call1() -declare i32 @call2() -declare i32 @call3() -declare i32 @call4(i32) -- GitLab From ebe77cc320a1bcc8e2cec44f4f388fb43b72016d Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Tue, 13 Feb 2024 18:00:11 +0800 Subject: [PATCH 038/284] [clang-tidy] ignore local variable with [maybe_unused] attribute in bugprone-unused-local-non-trivial-variable (#81563) --- .../bugprone/UnusedLocalNonTrivialVariableCheck.cpp | 1 + clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checks/bugprone/unused-local-non-trivial-variable.rst | 1 + .../checkers/bugprone/unused-local-non-trivial-variable.cpp | 1 + 4 files changed, 7 insertions(+) diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedLocalNonTrivialVariableCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedLocalNonTrivialVariableCheck.cpp index 1b763d291082..37baae7a6f0c 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedLocalNonTrivialVariableCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedLocalNonTrivialVariableCheck.cpp @@ -60,6 +60,7 @@ void UnusedLocalNonTrivialVariableCheck::registerMatchers(MatchFinder *Finder) { varDecl(isLocalVarDecl(), unless(isReferenced()), unless(isExceptionVariable()), hasLocalStorage(), isDefinition(), unless(hasType(isReferenceType())), unless(hasType(isTrivial())), + unless(hasAttr(attr::Kind::Unused)), hasType(hasUnqualifiedDesugaredType( anyOf(recordType(hasDeclaration(namedDecl( matchesAnyListedName(IncludeTypes), diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index ee68c8f49b3d..f2fba9aa1450 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -121,6 +121,10 @@ Changes in existing checks ` check by incorporating better support for ``const`` loop boundaries. +- Improved :doc:`bugprone-unused-local-non-trivial-variable + ` check by + ignoring local variable with ``[maybe_unused]`` attribute. + - Cleaned up :doc:`cppcoreguidelines-prefer-member-initializer ` by removing enforcement of rule `C.48 diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-local-non-trivial-variable.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-local-non-trivial-variable.rst index 7531f19f3ebc..9f283de78fbd 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-local-non-trivial-variable.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-local-non-trivial-variable.rst @@ -11,6 +11,7 @@ The following types of variables are excluded from this check: * exception variables in catch clauses * static or thread local * structured bindings +* variables with ``[[maybe_unused]]`` attribute This check can be configured to warn on all non-trivial variables by setting `IncludeTypes` to `.*`, and excluding specific types using `ExcludeTypes`. diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp index 19f2344de4a6..3fdc24b94a6c 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp @@ -77,6 +77,7 @@ T qux(T Generic) { // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: unused local variable 'TemplateType' of type 'async::Future' [bugprone-unused-local-non-trivial-variable] a::Future AliasTemplateType; // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: unused local variable 'AliasTemplateType' of type 'a::Future' (aka 'Future') [bugprone-unused-local-non-trivial-variable] + [[maybe_unused]] async::Future MaybeUnused; return Generic; } -- GitLab From 8c6e96d9eb35849762fa3ab4d3cc9517c4e14e74 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Tue, 13 Feb 2024 11:00:04 +0100 Subject: [PATCH 039/284] [clang][Dataflow] Fix unnecessary copy in `initializeFieldsWithValues` (NFC) --- clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index 93a9dac3bc90..d487944ce921 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -939,7 +939,7 @@ void Environment::initializeFieldsWithValues(RecordStorageLocation &Loc, } }; - for (const auto [Field, FieldLoc] : Loc.children()) { + for (const auto &[Field, FieldLoc] : Loc.children()) { assert(Field != nullptr); QualType FieldType = Field->getType(); -- GitLab From f506192c016fb6909eb2c17e183f6223db012f8c Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Tue, 13 Feb 2024 11:17:28 +0100 Subject: [PATCH 040/284] [libc][NFC] Small `abs` related simplifications (#79858) --- libc/src/__support/FPUtil/BasicOperations.h | 4 +--- libc/src/math/generic/acoshf.cpp | 6 +----- libc/src/math/generic/asinhf.cpp | 4 +--- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h index ccc61a89c5f8..a19d6d0bef08 100644 --- a/libc/src/__support/FPUtil/BasicOperations.h +++ b/libc/src/__support/FPUtil/BasicOperations.h @@ -19,9 +19,7 @@ namespace fputil { template , int> = 0> LIBC_INLINE T abs(T x) { - FPBits bits(x); - bits.set_sign(Sign::POS); - return bits.get_val(); + return FPBits(x).abs().get_val(); } template , int> = 0> diff --git a/libc/src/math/generic/acoshf.cpp b/libc/src/math/generic/acoshf.cpp index 54b66bf42fc6..a4a75a7b0438 100644 --- a/libc/src/math/generic/acoshf.cpp +++ b/libc/src/math/generic/acoshf.cpp @@ -33,12 +33,8 @@ LLVM_LIBC_FUNCTION(float, acoshf, (float x)) { } if (LIBC_UNLIKELY(x_u >= 0x4f8ffb03)) { - // Check for exceptional values. - uint32_t x_abs = xbits.abs().uintval(); - if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { - // x is +inf or NaN. + if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) return x; - } // Helper functions to set results for exceptional cases. auto round_result_slightly_down = [](float r) -> float { diff --git a/libc/src/math/generic/asinhf.cpp b/libc/src/math/generic/asinhf.cpp index ac059910b4ef..6e351786e3ec 100644 --- a/libc/src/math/generic/asinhf.cpp +++ b/libc/src/math/generic/asinhf.cpp @@ -59,10 +59,8 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { }; if (LIBC_UNLIKELY(x_abs >= 0x4bdd'65a5U)) { - if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { - // x is +-inf or nan + if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) return x; - } // Exceptional cases when x > 2^24. switch (x_abs) { -- GitLab From 4c931091a344e3284bd449710defdabe8fab1c22 Mon Sep 17 00:00:00 2001 From: Ivan Kosarev Date: Tue, 13 Feb 2024 12:27:56 +0200 Subject: [PATCH 041/284] [AMDGPU][NFC] Get rid of some operand decoders defined using macros. (#81482) Use templates instead. Part of . --- .../Disassembler/AMDGPUDisassembler.cpp | 84 +++------ llvm/lib/Target/AMDGPU/SIRegisterInfo.td | 165 +++++++++--------- 2 files changed, 111 insertions(+), 138 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp index 85377d07c52d..b307865275d0 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -178,8 +178,12 @@ static DecodeStatus decodeAV10(MCInst &Inst, unsigned Imm, uint64_t /* Addr */, } // Decoder for Src(9-bit encoding) registers only. -#define DECODE_OPERAND_SRC_REG_9(RegClass, OpWidth) \ - DECODE_SrcOp(decodeOperand_##RegClass, 9, OpWidth, Imm, false, 0) +template +static DecodeStatus decodeSrcReg9(MCInst &Inst, unsigned Imm, + uint64_t /* Addr */, + const MCDisassembler *Decoder) { + return decodeSrcOp(Inst, 9, OpWidth, Imm, Imm, false, 0, Decoder); +} // Decoder for Src(9-bit encoding) AGPR, register number encoded in 9bits, set // Imm{9} to 1 (set acc) and decode using 'enum10' from decodeSrcOp, registers @@ -204,22 +208,29 @@ static DecodeStatus decodeSrcAV10(MCInst &Inst, unsigned Imm, // will be decoded and InstPrinter will report warning. Immediate will be // decoded into constant of size ImmWidth, should match width of immediate used // by OperandType (important for floating point types). -#define DECODE_OPERAND_SRC_REG_OR_IMM_9(RegClass, OpWidth, ImmWidth) \ - DECODE_SrcOp(decodeOperand_##RegClass##_Imm##ImmWidth, 9, OpWidth, Imm, \ - false, ImmWidth) - -#define DECODE_OPERAND_SRC_REG_OR_IMM_9_TYPED(Name, OpWidth, ImmWidth) \ - DECODE_SrcOp(decodeOperand_##Name, 9, OpWidth, Imm, false, ImmWidth) +template +static DecodeStatus decodeSrcRegOrImm9(MCInst &Inst, unsigned Imm, + uint64_t /* Addr */, + const MCDisassembler *Decoder) { + return decodeSrcOp(Inst, 9, OpWidth, Imm, Imm, false, ImmWidth, Decoder); +} // Decoder for Src(9-bit encoding) AGPR or immediate. Set Imm{9} to 1 (set acc) // and decode using 'enum10' from decodeSrcOp. -#define DECODE_OPERAND_SRC_REG_OR_IMM_A9(RegClass, OpWidth, ImmWidth) \ - DECODE_SrcOp(decodeOperand_##RegClass##_Imm##ImmWidth, 9, OpWidth, \ - Imm | 512, false, ImmWidth) +template +static DecodeStatus decodeSrcRegOrImmA9(MCInst &Inst, unsigned Imm, + uint64_t /* Addr */, + const MCDisassembler *Decoder) { + return decodeSrcOp(Inst, 9, OpWidth, Imm, Imm | 512, false, ImmWidth, + Decoder); +} -#define DECODE_OPERAND_SRC_REG_OR_IMM_DEFERRED_9(RegClass, OpWidth, ImmWidth) \ - DECODE_SrcOp(decodeOperand_##RegClass##_Deferred##_Imm##ImmWidth, 9, \ - OpWidth, Imm, true, ImmWidth) +template +static DecodeStatus decodeSrcRegOrImmDeferred9(MCInst &Inst, unsigned Imm, + uint64_t /* Addr */, + const MCDisassembler *Decoder) { + return decodeSrcOp(Inst, 9, OpWidth, Imm, Imm, true, ImmWidth, Decoder); +} // Default decoders generated by tablegen: 'DecodeRegisterClass' // when RegisterClass is used as an operand. Most often used for destination @@ -255,51 +266,6 @@ DECODE_OPERAND_REG_8(AReg_256) DECODE_OPERAND_REG_8(AReg_512) DECODE_OPERAND_REG_8(AReg_1024) -// Decoders for register only source RegisterOperands that use use 9-bit Src -// encoding: 'decodeOperand_'. - -DECODE_OPERAND_SRC_REG_9(VGPR_32, OPW32) -DECODE_OPERAND_SRC_REG_9(VReg_64, OPW64) -DECODE_OPERAND_SRC_REG_9(VReg_128, OPW128) -DECODE_OPERAND_SRC_REG_9(VReg_256, OPW256) -DECODE_OPERAND_SRC_REG_9(VRegOrLds_32, OPW32) - -// Decoders for register or immediate RegisterOperands that use 9-bit Src -// encoding: 'decodeOperand__Imm'. - -DECODE_OPERAND_SRC_REG_OR_IMM_9(SReg_64, OPW64, 64) -DECODE_OPERAND_SRC_REG_OR_IMM_9(SReg_32, OPW32, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(SReg_32, OPW32, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_9(SRegOrLds_32, OPW32, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VS_32_Lo128, OPW16, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VS_32, OPW32, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VS_32, OPW32, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VS_64, OPW64, 64) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VS_64, OPW64, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_64, OPW64, 64) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_64, OPW64, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_64, OPW64, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_128, OPW128, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_128, OPW128, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_256, OPW256, 64) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_256, OPW256, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_512, OPW512, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9(VReg_1024, OPW1024, 32) - -DECODE_OPERAND_SRC_REG_OR_IMM_9_TYPED(VS_32_ImmV2I16, OPW32, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_9_TYPED(VS_32_ImmV2F16, OPW32, 16) - -DECODE_OPERAND_SRC_REG_OR_IMM_A9(AReg_64, OPW64, 64) -DECODE_OPERAND_SRC_REG_OR_IMM_A9(AReg_128, OPW128, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_A9(AReg_256, OPW256, 64) -DECODE_OPERAND_SRC_REG_OR_IMM_A9(AReg_512, OPW512, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_A9(AReg_1024, OPW1024, 32) - -DECODE_OPERAND_SRC_REG_OR_IMM_DEFERRED_9(VS_32_Lo128, OPW16, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_DEFERRED_9(VS_32, OPW16, 16) -DECODE_OPERAND_SRC_REG_OR_IMM_DEFERRED_9(VS_32, OPW32, 32) -DECODE_OPERAND_SRC_REG_OR_IMM_DEFERRED_9(SReg_32, OPW32, 32) - static DecodeStatus DecodeVGPR_16RegisterClass(MCInst &Inst, unsigned Imm, uint64_t /*Addr*/, const MCDisassembler *Decoder) { diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td index aabb6c290621..d4a1e8d185a1 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td @@ -1101,96 +1101,106 @@ class RegImmMatcher : AsmOperandClass { let RenderMethod = "addRegOrImmOperands"; } -class RegOrImmOperand - : RegisterOperand(RegisterClassName)> { +class RegOrImmOperand + : RegisterOperand { let OperandNamespace = "AMDGPU"; let OperandType = OperandTypeName; let ParserMatchClass = RegImmMatcher; - let DecoderMethod = "decodeOperand_" # RegisterClassName # decoderImmSize; } //===----------------------------------------------------------------------===// // SSrc_* Operands with an SGPR or a 32-bit immediate //===----------------------------------------------------------------------===// -def SSrc_b16 : RegOrImmOperand <"SReg_32", "OPERAND_REG_IMM_INT16", "_Imm16">; -def SSrc_f16 : RegOrImmOperand <"SReg_32", "OPERAND_REG_IMM_FP16", "_Imm16">; -def SSrc_b32 : RegOrImmOperand <"SReg_32", "OPERAND_REG_IMM_INT32", "_Imm32">; -def SSrc_f32 : RegOrImmOperand <"SReg_32", "OPERAND_REG_IMM_FP32", "_Imm32">; -def SSrc_b64 : RegOrImmOperand <"SReg_64", "OPERAND_REG_IMM_INT64", "_Imm64">; +class SrcRegOrImm9 : RegOrImmOperand { + let DecoderMethod = "decodeSrcRegOrImm9"; +} + +def SSrc_b16 : SrcRegOrImm9 ; +def SSrc_f16 : SrcRegOrImm9 ; +def SSrc_b32 : SrcRegOrImm9 ; +def SSrc_f32 : SrcRegOrImm9 ; +def SSrc_b64 : SrcRegOrImm9 ; -def SSrcOrLds_b32 : RegOrImmOperand <"SRegOrLds_32", "OPERAND_REG_IMM_INT32", "_Imm32">; +def SSrcOrLds_b32 : SrcRegOrImm9 ; //===----------------------------------------------------------------------===// // SSrc_32_Deferred Operands with an SGPR or a 32-bit immediate for use with // FMAMK/FMAAK //===----------------------------------------------------------------------===// -def SSrc_f32_Deferred : RegOrImmOperand<"SReg_32", "OPERAND_REG_IMM_FP32_DEFERRED", "_Deferred_Imm32">; +class SrcRegOrImmDeferred9 + : RegOrImmOperand { + let DecoderMethod = "decodeSrcRegOrImmDeferred9"; +} + +def SSrc_f32_Deferred : SrcRegOrImmDeferred9; //===----------------------------------------------------------------------===// // SCSrc_* Operands with an SGPR or a inline constant //===----------------------------------------------------------------------===// -def SCSrc_b32 : RegOrImmOperand <"SReg_32", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def SCSrc_b64 : RegOrImmOperand <"SReg_64", "OPERAND_REG_INLINE_C_INT64", "_Imm64">; +def SCSrc_b32 : SrcRegOrImm9 ; +def SCSrc_b64 : SrcRegOrImm9 ; //===----------------------------------------------------------------------===// // VSrc_* Operands with an SGPR, VGPR or a 32-bit immediate //===----------------------------------------------------------------------===// // The current and temporary future default used case for VOP3. -def VSrc_b16 : RegOrImmOperand <"VS_32", "OPERAND_REG_IMM_INT16", "_Imm16">; -def VSrc_f16 : RegOrImmOperand <"VS_32", "OPERAND_REG_IMM_FP16", "_Imm16">; +def VSrc_b16 : SrcRegOrImm9 ; +def VSrc_f16 : SrcRegOrImm9 ; // True16 VOP3 operands. -def VSrcT_b16 : RegOrImmOperand <"VS_16", "OPERAND_REG_IMM_INT16", "_Imm16"> { +def VSrcT_b16 : RegOrImmOperand { let EncoderMethod = "getMachineOpValueT16"; let DecoderMethod = "decodeOperand_VSrcT16"; } -def VSrcT_f16 : RegOrImmOperand <"VS_16", "OPERAND_REG_IMM_FP16", "_Imm16"> { +def VSrcT_f16 : RegOrImmOperand { let EncoderMethod = "getMachineOpValueT16"; let DecoderMethod = "decodeOperand_VSrcT16"; } // True16 VOP1/2/C operands. -def VSrcT_b16_Lo128 : RegOrImmOperand <"VS_16_Lo128", "OPERAND_REG_IMM_INT16", "_Imm16"> { +def VSrcT_b16_Lo128 : RegOrImmOperand { let EncoderMethod = "getMachineOpValueT16Lo128"; let DecoderMethod = "decodeOperand_VSrcT16_Lo128"; } -def VSrcT_f16_Lo128 : RegOrImmOperand <"VS_16_Lo128", "OPERAND_REG_IMM_FP16", "_Imm16"> { +def VSrcT_f16_Lo128 : RegOrImmOperand { let EncoderMethod = "getMachineOpValueT16Lo128"; let DecoderMethod = "decodeOperand_VSrcT16_Lo128"; } // The current and temporary future default used case for fake VOP1/2/C. // For VOP1,2,C True16 instructions. _Lo128 use first 128 32-bit VGPRs only. -def VSrcFake16_b16_Lo128 : RegOrImmOperand <"VS_32_Lo128", "OPERAND_REG_IMM_INT16", "_Imm16">; -def VSrcFake16_f16_Lo128 : RegOrImmOperand <"VS_32_Lo128", "OPERAND_REG_IMM_FP16", "_Imm16">; - -def VSrc_b32 : RegOrImmOperand <"VS_32", "OPERAND_REG_IMM_INT32", "_Imm32">; -def VSrc_f32 : RegOrImmOperand <"VS_32", "OPERAND_REG_IMM_FP32", "_Imm32">; -def VSrc_v2b16 : RegOrImmOperand <"VS_32", "OPERAND_REG_IMM_V2INT16", "_ImmV2I16">; -def VSrc_v2f16 : RegOrImmOperand <"VS_32", "OPERAND_REG_IMM_V2FP16", "_ImmV2F16">; -def VSrc_b64 : RegOrImmOperand <"VS_64", "OPERAND_REG_IMM_INT64", "_Imm64">; -def VSrc_f64 : RegOrImmOperand <"VS_64", "OPERAND_REG_IMM_FP64", "_Imm64"> { +def VSrcFake16_b16_Lo128 : SrcRegOrImm9 ; +def VSrcFake16_f16_Lo128 : SrcRegOrImm9 ; + +def VSrc_b32 : SrcRegOrImm9 ; +def VSrc_f32 : SrcRegOrImm9 ; +def VSrc_v2b16 : SrcRegOrImm9 ; +def VSrc_v2f16 : SrcRegOrImm9 ; +def VSrc_b64 : SrcRegOrImm9 ; +def VSrc_f64 : SrcRegOrImm9 { let DecoderMethod = "decodeOperand_VSrc_f64"; } -def VSrc_v2b32 : RegOrImmOperand <"VS_64", "OPERAND_REG_IMM_V2INT32", "_Imm32">; -def VSrc_v2f32 : RegOrImmOperand <"VS_64", "OPERAND_REG_IMM_V2FP32", "_Imm32">; +def VSrc_v2b32 : SrcRegOrImm9 ; +def VSrc_v2f32 : SrcRegOrImm9 ; //===----------------------------------------------------------------------===// // VSrc_*_Deferred Operands with an SGPR, VGPR or a 32-bit immediate for use // with FMAMK/FMAAK //===----------------------------------------------------------------------===// -def VSrc_f16_Deferred : RegOrImmOperand<"VS_32", "OPERAND_REG_IMM_FP16_DEFERRED", "_Deferred_Imm16">; -def VSrc_f32_Deferred : RegOrImmOperand<"VS_32", "OPERAND_REG_IMM_FP32_DEFERRED", "_Deferred_Imm32">; +def VSrc_f16_Deferred : SrcRegOrImmDeferred9; +def VSrc_f32_Deferred : SrcRegOrImmDeferred9; -def VSrcFake16_f16_Lo128_Deferred : RegOrImmOperand<"VS_32_Lo128", - "OPERAND_REG_IMM_FP16_DEFERRED", - "_Deferred_Imm16">; +def VSrcFake16_f16_Lo128_Deferred + : SrcRegOrImmDeferred9; //===----------------------------------------------------------------------===// // VRegSrc_* Operands with a VGPR @@ -1198,25 +1208,15 @@ def VSrcFake16_f16_Lo128_Deferred : RegOrImmOperand<"VS_32_Lo128", // This is for operands with the enum(9), VSrc encoding restriction, // but only allows VGPRs. -def VRegSrc_32 : RegisterOperand { - let DecoderMethod = "decodeOperand_VGPR_32"; -} - -def VRegSrc_64 : RegisterOperand { - let DecoderMethod = "decodeOperand_VReg_64"; +class SrcReg9 : RegisterOperand { + let DecoderMethod = "decodeSrcReg9"; } -def VRegSrc_128 : RegisterOperand { - let DecoderMethod = "decodeOperand_VReg_128"; -} - -def VRegSrc_256 : RegisterOperand { - let DecoderMethod = "decodeOperand_VReg_256"; -} - -def VRegOrLdsSrc_32 : RegisterOperand { - let DecoderMethod = "decodeOperand_VRegOrLds_32"; -} +def VRegSrc_32 : SrcReg9; +def VRegSrc_64 : SrcReg9; +def VRegSrc_128: SrcReg9; +def VRegSrc_256: SrcReg9; +def VRegOrLdsSrc_32 : SrcReg9; //===----------------------------------------------------------------------===// // VGPRSrc_* @@ -1257,30 +1257,30 @@ def ARegSrc_32 : AVOperand; // VCSrc_* Operands with an SGPR, VGPR or an inline constant //===----------------------------------------------------------------------===// -def VCSrc_b16 : RegOrImmOperand <"VS_32", "OPERAND_REG_INLINE_C_INT16", "_Imm16">; -def VCSrc_f16 : RegOrImmOperand <"VS_32", "OPERAND_REG_INLINE_C_FP16", "_Imm16">; -def VCSrc_b32 : RegOrImmOperand <"VS_32", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def VCSrc_f32 : RegOrImmOperand <"VS_32", "OPERAND_REG_INLINE_C_FP32", "_Imm32">; -def VCSrc_v2b16 : RegOrImmOperand <"VS_32", "OPERAND_REG_INLINE_C_V2INT16", "_ImmV2I16">; -def VCSrc_v2f16 : RegOrImmOperand <"VS_32", "OPERAND_REG_INLINE_C_V2FP16", "_ImmV2F16">; +def VCSrc_b16 : SrcRegOrImm9 ; +def VCSrc_f16 : SrcRegOrImm9 ; +def VCSrc_b32 : SrcRegOrImm9 ; +def VCSrc_f32 : SrcRegOrImm9 ; +def VCSrc_v2b16 : SrcRegOrImm9 ; +def VCSrc_v2f16 : SrcRegOrImm9 ; //===----------------------------------------------------------------------===// // VISrc_* Operands with a VGPR or an inline constant //===----------------------------------------------------------------------===// -def VISrc_64_f16 : RegOrImmOperand <"VReg_64", "OPERAND_REG_INLINE_C_FP16", "_Imm16">; -def VISrc_64_b32 : RegOrImmOperand <"VReg_64", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def VISrc_64_f64 : RegOrImmOperand <"VReg_64", "OPERAND_REG_INLINE_C_FP64", "_Imm64">; -def VISrc_128_f16 : RegOrImmOperand <"VReg_128", "OPERAND_REG_INLINE_C_FP16", "_Imm16">; -def VISrc_128_b32 : RegOrImmOperand <"VReg_128", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def VISrc_128_f32 : RegOrImmOperand <"VReg_128", "OPERAND_REG_INLINE_C_FP32", "_Imm32">; -def VISrc_256_b32 : RegOrImmOperand <"VReg_256", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def VISrc_256_f32 : RegOrImmOperand <"VReg_256", "OPERAND_REG_INLINE_C_FP32", "_Imm32">; -def VISrc_256_f64 : RegOrImmOperand <"VReg_256", "OPERAND_REG_INLINE_C_FP64", "_Imm64">; -def VISrc_512_b32 : RegOrImmOperand <"VReg_512", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def VISrc_512_f32 : RegOrImmOperand <"VReg_512", "OPERAND_REG_INLINE_C_FP32", "_Imm32">; -def VISrc_1024_b32 : RegOrImmOperand <"VReg_1024", "OPERAND_REG_INLINE_C_INT32", "_Imm32">; -def VISrc_1024_f32 : RegOrImmOperand <"VReg_1024", "OPERAND_REG_INLINE_C_FP32", "_Imm32">; +def VISrc_64_f16 : SrcRegOrImm9 ; +def VISrc_64_b32 : SrcRegOrImm9 ; +def VISrc_64_f64 : SrcRegOrImm9 ; +def VISrc_128_f16 : SrcRegOrImm9 ; +def VISrc_128_b32 : SrcRegOrImm9 ; +def VISrc_128_f32 : SrcRegOrImm9 ; +def VISrc_256_b32 : SrcRegOrImm9 ; +def VISrc_256_f32 : SrcRegOrImm9 ; +def VISrc_256_f64 : SrcRegOrImm9 ; +def VISrc_512_b32 : SrcRegOrImm9 ; +def VISrc_512_f32 : SrcRegOrImm9 ; +def VISrc_1024_b32 : SrcRegOrImm9 ; +def VISrc_1024_f32 : SrcRegOrImm9 ; //===----------------------------------------------------------------------===// // AVSrc_*, AVDst_*, AVLdSt_* Operands with an AGPR or VGPR @@ -1312,11 +1312,18 @@ def AVLdSt_160 : AVLdStOperand; // ACSrc_* Operands with an AGPR or an inline constant //===----------------------------------------------------------------------===// -def AISrc_64_f64 : RegOrImmOperand <"AReg_64", "OPERAND_REG_INLINE_AC_FP64", "_Imm64">; -def AISrc_128_f32 : RegOrImmOperand <"AReg_128", "OPERAND_REG_INLINE_AC_FP32", "_Imm32">; -def AISrc_128_b32 : RegOrImmOperand <"AReg_128", "OPERAND_REG_INLINE_AC_INT32", "_Imm32">; -def AISrc_256_f64 : RegOrImmOperand <"AReg_256", "OPERAND_REG_INLINE_AC_FP64", "_Imm64">; -def AISrc_512_f32 : RegOrImmOperand <"AReg_512", "OPERAND_REG_INLINE_AC_FP32", "_Imm32">; -def AISrc_512_b32 : RegOrImmOperand <"AReg_512", "OPERAND_REG_INLINE_AC_INT32", "_Imm32">; -def AISrc_1024_f32 : RegOrImmOperand <"AReg_1024", "OPERAND_REG_INLINE_AC_FP32", "_Imm32">; -def AISrc_1024_b32 : RegOrImmOperand <"AReg_1024", "OPERAND_REG_INLINE_AC_INT32", "_Imm32">; +class SrcRegOrImmA9 + : RegOrImmOperand { + let DecoderMethod = "decodeSrcRegOrImmA9"; +} + +def AISrc_64_f64 : SrcRegOrImmA9 ; +def AISrc_128_f32 : SrcRegOrImmA9 ; +def AISrc_128_b32 : SrcRegOrImmA9 ; +def AISrc_256_f64 : SrcRegOrImmA9 ; +def AISrc_512_f32 : SrcRegOrImmA9 ; +def AISrc_512_b32 : SrcRegOrImmA9 ; +def AISrc_1024_f32 : SrcRegOrImmA9 ; +def AISrc_1024_b32 : SrcRegOrImmA9 ; -- GitLab From fe3406e349884e4ef61480dd0607f1e237102c74 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Tue, 13 Feb 2024 11:29:21 +0100 Subject: [PATCH 042/284] [lld] Add target support for SystemZ (s390x) (#75643) This patch adds full support for linking SystemZ (ELF s390x) object files. Support should be generally complete: - All relocation types are supported. - Full shared library support (DYNAMIC, GOT, PLT, ifunc). - Relaxation of TLS and GOT relocations where appropriate. - Platform-specific test cases. In addition to new platform code and the obvious changes, there were a few additional changes to common code: - Add three new RelExpr members (R_GOTPLT_OFF, R_GOTPLT_PC, and R_PLT_GOTREL) needed to support certain s390x relocations. I chose not to use a platform-specific name since nothing in the definition of these relocs is actually platform-specific; it is well possible that other platforms will need the same. - A couple of tweaks to TLS relocation handling, as the particular semantics of the s390x versions differ slightly. See comments in the code. This was tested by building and testing >1500 Fedora packages, with only a handful of failures; as these also have issues when building with LLD on other architectures, they seem unrelated. Co-authored-by: Tulio Magno Quites Machado Filho --- lld/ELF/Arch/SystemZ.cpp | 607 ++++++++++++++++++++ lld/ELF/CMakeLists.txt | 1 + lld/ELF/Driver.cpp | 3 +- lld/ELF/InputFiles.cpp | 2 + lld/ELF/InputSection.cpp | 7 + lld/ELF/Relocations.cpp | 25 +- lld/ELF/Relocations.h | 3 + lld/ELF/ScriptParser.cpp | 1 + lld/ELF/SyntheticSections.cpp | 3 + lld/ELF/Target.cpp | 2 + lld/ELF/Target.h | 1 + lld/test/ELF/Inputs/systemz-init.s | 5 + lld/test/ELF/basic-systemz.s | 63 ++ lld/test/ELF/emulation-systemz.s | 29 + lld/test/ELF/lto/systemz.ll | 18 + lld/test/ELF/systemz-got.s | 16 + lld/test/ELF/systemz-gotent-relax-align.s | 48 ++ lld/test/ELF/systemz-gotent-relax-und-dso.s | 68 +++ lld/test/ELF/systemz-gotent-relax.s | 91 +++ lld/test/ELF/systemz-ifunc-nonpreemptible.s | 75 +++ lld/test/ELF/systemz-init-padding.s | 27 + lld/test/ELF/systemz-pie.s | 38 ++ lld/test/ELF/systemz-plt.s | 83 +++ lld/test/ELF/systemz-reloc-abs.s | 32 ++ lld/test/ELF/systemz-reloc-disp12.s | 21 + lld/test/ELF/systemz-reloc-disp20.s | 21 + lld/test/ELF/systemz-reloc-got.s | 92 +++ lld/test/ELF/systemz-reloc-gotrel.s | 36 ++ lld/test/ELF/systemz-reloc-pc16.s | 39 ++ lld/test/ELF/systemz-reloc-pc32.s | 39 ++ lld/test/ELF/systemz-reloc-pcdbl.s | 68 +++ lld/test/ELF/systemz-tls-gd.s | 142 +++++ lld/test/ELF/systemz-tls-ie.s | 87 +++ lld/test/ELF/systemz-tls-ld.s | 114 ++++ lld/test/ELF/systemz-tls-le.s | 61 ++ lld/test/lit.cfg.py | 1 + 36 files changed, 1959 insertions(+), 10 deletions(-) create mode 100644 lld/ELF/Arch/SystemZ.cpp create mode 100644 lld/test/ELF/Inputs/systemz-init.s create mode 100644 lld/test/ELF/basic-systemz.s create mode 100644 lld/test/ELF/emulation-systemz.s create mode 100644 lld/test/ELF/lto/systemz.ll create mode 100644 lld/test/ELF/systemz-got.s create mode 100644 lld/test/ELF/systemz-gotent-relax-align.s create mode 100644 lld/test/ELF/systemz-gotent-relax-und-dso.s create mode 100644 lld/test/ELF/systemz-gotent-relax.s create mode 100644 lld/test/ELF/systemz-ifunc-nonpreemptible.s create mode 100644 lld/test/ELF/systemz-init-padding.s create mode 100644 lld/test/ELF/systemz-pie.s create mode 100644 lld/test/ELF/systemz-plt.s create mode 100644 lld/test/ELF/systemz-reloc-abs.s create mode 100644 lld/test/ELF/systemz-reloc-disp12.s create mode 100644 lld/test/ELF/systemz-reloc-disp20.s create mode 100644 lld/test/ELF/systemz-reloc-got.s create mode 100644 lld/test/ELF/systemz-reloc-gotrel.s create mode 100644 lld/test/ELF/systemz-reloc-pc16.s create mode 100644 lld/test/ELF/systemz-reloc-pc32.s create mode 100644 lld/test/ELF/systemz-reloc-pcdbl.s create mode 100644 lld/test/ELF/systemz-tls-gd.s create mode 100644 lld/test/ELF/systemz-tls-ie.s create mode 100644 lld/test/ELF/systemz-tls-ld.s create mode 100644 lld/test/ELF/systemz-tls-le.s diff --git a/lld/ELF/Arch/SystemZ.cpp b/lld/ELF/Arch/SystemZ.cpp new file mode 100644 index 000000000000..d37db6877559 --- /dev/null +++ b/lld/ELF/Arch/SystemZ.cpp @@ -0,0 +1,607 @@ +//===- SystemZ.cpp --------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OutputSections.h" +#include "Symbols.h" +#include "SyntheticSections.h" +#include "Target.h" +#include "lld/Common/ErrorHandler.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Support/Endian.h" + +using namespace llvm; +using namespace llvm::support::endian; +using namespace llvm::ELF; +using namespace lld; +using namespace lld::elf; + +namespace { +class SystemZ : public TargetInfo { +public: + SystemZ(); + int getTlsGdRelaxSkip(RelType type) const override; + RelExpr getRelExpr(RelType type, const Symbol &s, + const uint8_t *loc) const override; + RelType getDynRel(RelType type) const override; + void writeGotHeader(uint8_t *buf) const override; + void writeGotPlt(uint8_t *buf, const Symbol &s) const override; + void writeIgotPlt(uint8_t *buf, const Symbol &s) const override; + void writePltHeader(uint8_t *buf) const override; + void addPltHeaderSymbols(InputSection &isd) const override; + void writePlt(uint8_t *buf, const Symbol &sym, + uint64_t pltEntryAddr) const override; + RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; + RelExpr adjustGotPcExpr(RelType type, int64_t addend, + const uint8_t *loc) const override; + bool relaxOnce(int pass) const override; + void relocate(uint8_t *loc, const Relocation &rel, + uint64_t val) const override; + int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; + +private: + void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; +}; +} // namespace + +SystemZ::SystemZ() { + copyRel = R_390_COPY; + gotRel = R_390_GLOB_DAT; + pltRel = R_390_JMP_SLOT; + relativeRel = R_390_RELATIVE; + iRelativeRel = R_390_IRELATIVE; + symbolicRel = R_390_64; + tlsGotRel = R_390_TLS_TPOFF; + tlsModuleIndexRel = R_390_TLS_DTPMOD; + tlsOffsetRel = R_390_TLS_DTPOFF; + gotHeaderEntriesNum = 3; + gotPltHeaderEntriesNum = 0; + gotEntrySize = 8; + pltHeaderSize = 32; + pltEntrySize = 32; + ipltEntrySize = 32; + + // This "trap instruction" is used to fill gaps between sections. + // On SystemZ, the behavior of the GNU ld is to fill those gaps + // with nop instructions instead - and unfortunately the default + // glibc crt object files (used to) rely on that behavior since + // they use an alignment on the .init section fragments that causes + // gaps which must be filled with nops as they are being executed. + // Therefore, we provide a nop instruction as "trapInstr" here. + trapInstr = {0x07, 0x07, 0x07, 0x07}; + + defaultImageBase = 0x1000000; +} + +RelExpr SystemZ::getRelExpr(RelType type, const Symbol &s, + const uint8_t *loc) const { + switch (type) { + case R_390_NONE: + return R_NONE; + // Relocations targeting the symbol value. + case R_390_8: + case R_390_12: + case R_390_16: + case R_390_20: + case R_390_32: + case R_390_64: + return R_ABS; + case R_390_PC16: + case R_390_PC32: + case R_390_PC64: + case R_390_PC12DBL: + case R_390_PC16DBL: + case R_390_PC24DBL: + case R_390_PC32DBL: + return R_PC; + case R_390_GOTOFF16: + case R_390_GOTOFF: // a.k.a. R_390_GOTOFF32 + case R_390_GOTOFF64: + return R_GOTREL; + // Relocations targeting the PLT associated with the symbol. + case R_390_PLT32: + case R_390_PLT64: + case R_390_PLT12DBL: + case R_390_PLT16DBL: + case R_390_PLT24DBL: + case R_390_PLT32DBL: + return R_PLT_PC; + case R_390_PLTOFF16: + case R_390_PLTOFF32: + case R_390_PLTOFF64: + return R_PLT_GOTREL; + // Relocations targeting the GOT entry associated with the symbol. + case R_390_GOTENT: + return R_GOT_PC; + case R_390_GOT12: + case R_390_GOT16: + case R_390_GOT20: + case R_390_GOT32: + case R_390_GOT64: + return R_GOT_OFF; + // Relocations targeting the GOTPLT entry associated with the symbol. + case R_390_GOTPLTENT: + return R_GOTPLT_PC; + case R_390_GOTPLT12: + case R_390_GOTPLT16: + case R_390_GOTPLT20: + case R_390_GOTPLT32: + case R_390_GOTPLT64: + return R_GOTPLT_GOTREL; + // Relocations targeting _GLOBAL_OFFSET_TABLE_. + case R_390_GOTPC: + case R_390_GOTPCDBL: + return R_GOTONLY_PC; + // TLS-related relocations. + case R_390_TLS_LOAD: + return R_NONE; + case R_390_TLS_GDCALL: + return R_TLSGD_PC; + case R_390_TLS_LDCALL: + return R_TLSLD_PC; + case R_390_TLS_GD32: + case R_390_TLS_GD64: + return R_TLSGD_GOT; + case R_390_TLS_LDM32: + case R_390_TLS_LDM64: + return R_TLSLD_GOT; + case R_390_TLS_LDO32: + case R_390_TLS_LDO64: + return R_DTPREL; + case R_390_TLS_LE32: + case R_390_TLS_LE64: + return R_TPREL; + case R_390_TLS_IE32: + case R_390_TLS_IE64: + return R_GOT; + case R_390_TLS_GOTIE12: + case R_390_TLS_GOTIE20: + case R_390_TLS_GOTIE32: + case R_390_TLS_GOTIE64: + return R_GOT_OFF; + case R_390_TLS_IEENT: + return R_GOT_PC; + + default: + error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + + ") against symbol " + toString(s)); + return R_NONE; + } +} + +void SystemZ::writeGotHeader(uint8_t *buf) const { + // _GLOBAL_OFFSET_TABLE_[0] holds the value of _DYNAMIC. + // _GLOBAL_OFFSET_TABLE_[1] and [2] are reserved. + write64be(buf, mainPart->dynamic->getVA()); +} + +void SystemZ::writeGotPlt(uint8_t *buf, const Symbol &s) const { + write64be(buf, s.getPltVA() + 14); +} + +void SystemZ::writeIgotPlt(uint8_t *buf, const Symbol &s) const { + if (config->writeAddends) + write64be(buf, s.getVA()); +} + +void SystemZ::writePltHeader(uint8_t *buf) const { + const uint8_t pltData[] = { + 0xe3, 0x10, 0xf0, 0x38, 0x00, 0x24, // stg %r1,56(%r15) + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, // larl %r1,_GLOBAL_OFFSET_TABLE_ + 0xd2, 0x07, 0xf0, 0x30, 0x10, 0x08, // mvc 48(8,%r15),8(%r1) + 0xe3, 0x10, 0x10, 0x10, 0x00, 0x04, // lg %r1,16(%r1) + 0x07, 0xf1, // br %r1 + 0x07, 0x00, // nopr + 0x07, 0x00, // nopr + 0x07, 0x00, // nopr + }; + memcpy(buf, pltData, sizeof(pltData)); + uint64_t got = in.got->getVA(); + uint64_t plt = in.plt->getVA(); + write32be(buf + 8, (got - plt - 6) >> 1); +} + +void SystemZ::addPltHeaderSymbols(InputSection &isec) const { + // The PLT header needs a reference to _GLOBAL_OFFSET_TABLE_, so we + // must ensure the .got section is created even if otherwise unused. + in.got->hasGotOffRel.store(true, std::memory_order_relaxed); +} + +void SystemZ::writePlt(uint8_t *buf, const Symbol &sym, + uint64_t pltEntryAddr) const { + const uint8_t inst[] = { + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, // larl %r1,<.got.plt slot> + 0xe3, 0x10, 0x10, 0x00, 0x00, 0x04, // lg %r1,0(%r1) + 0x07, 0xf1, // br %r1 + 0x0d, 0x10, // basr %r1,%r0 + 0xe3, 0x10, 0x10, 0x0c, 0x00, 0x14, // lgf %r1,12(%r1) + 0xc0, 0xf4, 0x00, 0x00, 0x00, 0x00, // jg + 0x00, 0x00, 0x00, 0x00, // + }; + memcpy(buf, inst, sizeof(inst)); + + write32be(buf + 2, (sym.getGotPltVA() - pltEntryAddr) >> 1); + write32be(buf + 24, (in.plt->getVA() - pltEntryAddr - 22) >> 1); + write32be(buf + 28, in.relaPlt->entsize * sym.getPltIdx()); +} + +int64_t SystemZ::getImplicitAddend(const uint8_t *buf, RelType type) const { + switch (type) { + case R_390_8: + return SignExtend64<8>(*buf); + case R_390_16: + case R_390_PC16: + return SignExtend64<16>(read16be(buf)); + case R_390_PC16DBL: + return SignExtend64<16>(read16be(buf)) << 1; + case R_390_32: + case R_390_PC32: + return SignExtend64<32>(read32be(buf)); + case R_390_PC32DBL: + return SignExtend64<32>(read32be(buf)) << 1; + case R_390_64: + case R_390_PC64: + case R_390_TLS_DTPMOD: + case R_390_TLS_DTPOFF: + case R_390_TLS_TPOFF: + case R_390_GLOB_DAT: + case R_390_RELATIVE: + case R_390_IRELATIVE: + return read64be(buf); + case R_390_COPY: + case R_390_JMP_SLOT: + case R_390_NONE: + // These relocations are defined as not having an implicit addend. + return 0; + default: + internalLinkerError(getErrorLocation(buf), + "cannot read addend for relocation " + toString(type)); + return 0; + } +} + +RelType SystemZ::getDynRel(RelType type) const { + if (type == R_390_64 || type == R_390_PC64) + return type; + return R_390_NONE; +} + +RelExpr SystemZ::adjustTlsExpr(RelType type, RelExpr expr) const { + if (expr == R_RELAX_TLS_GD_TO_IE) + return R_RELAX_TLS_GD_TO_IE_GOT_OFF; + return expr; +} + +int SystemZ::getTlsGdRelaxSkip(RelType type) const { + // A __tls_get_offset call instruction is marked with 2 relocations: + // + // R_390_TLS_GDCALL / R_390_TLS_LDCALL: marker relocation + // R_390_PLT32DBL: __tls_get_offset + // + // After the relaxation we no longer call __tls_get_offset and should skip + // both relocations to not create a false dependence on __tls_get_offset + // being defined. + // + // Note that this mechanism only works correctly if the R_390_TLS_[GL]DCALL + // is seen immediately *before* the R_390_PLT32DBL. Unfortunately, current + // compilers on the platform will typically generate the inverse sequence. + // To fix this, we sort relocations by offset in RelocationScanner::scan; + // this ensures the correct sequence as the R_390_TLS_[GL]DCALL applies to + // the first byte of the brasl instruction, while the R_390_PLT32DBL applies + // to its third byte (the relative displacement). + + if (type == R_390_TLS_GDCALL || type == R_390_TLS_LDCALL) + return 2; + return 1; +} + +void SystemZ::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, + uint64_t val) const { + // The general-dynamic code sequence for a global `x`: + // + // Instruction Relocation Symbol + // ear %rX,%a0 + // sllg %rX,%rX,32 + // ear %rX,%a1 + // larl %r12,_GLOBAL_OFFSET_TABLE_ R_390_GOTPCDBL _GLOBAL_OFFSET_TABLE_ + // lgrl %r2,.LC0 R_390_PC32DBL .LC0 + // brasl %r14,__tls_get_offset@plt R_390_TLS_GDCALL x + // :tls_gdcall:x R_390_PLT32DBL __tls_get_offset + // la %r2,0(%r2,%rX) + // + // .LC0: + // .quad x@TLSGD R_390_TLS_GD64 x + // + // Relaxing to initial-exec entails: + // 1) Replacing the call by a load from the GOT. + // 2) Replacing the relocation on the constant LC0 by R_390_TLS_GOTIE64. + + switch (rel.type) { + case R_390_TLS_GDCALL: + // brasl %r14,__tls_get_offset@plt -> lg %r2,0(%r2,%r12) + write16be(loc, 0xe322); + write32be(loc + 2, 0xc0000004); + break; + case R_390_TLS_GD64: + relocateNoSym(loc, R_390_TLS_GOTIE64, val); + break; + default: + llvm_unreachable("unsupported relocation for TLS GD to IE relaxation"); + } +} + +void SystemZ::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, + uint64_t val) const { + // The general-dynamic code sequence for a global `x`: + // + // Instruction Relocation Symbol + // ear %rX,%a0 + // sllg %rX,%rX,32 + // ear %rX,%a1 + // larl %r12,_GLOBAL_OFFSET_TABLE_ R_390_GOTPCDBL _GLOBAL_OFFSET_TABLE_ + // lgrl %r2,.LC0 R_390_PC32DBL .LC0 + // brasl %r14,__tls_get_offset@plt R_390_TLS_GDCALL x + // :tls_gdcall:x R_390_PLT32DBL __tls_get_offset + // la %r2,0(%r2,%rX) + // + // .LC0: + // .quad x@tlsgd R_390_TLS_GD64 x + // + // Relaxing to local-exec entails: + // 1) Replacing the call by a nop. + // 2) Replacing the relocation on the constant LC0 by R_390_TLS_LE64. + + switch (rel.type) { + case R_390_TLS_GDCALL: + // brasl %r14,__tls_get_offset@plt -> brcl 0,. + write16be(loc, 0xc004); + write32be(loc + 2, 0x00000000); + break; + case R_390_TLS_GD64: + relocateNoSym(loc, R_390_TLS_LE64, val); + break; + default: + llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); + } +} + +void SystemZ::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, + uint64_t val) const { + // The local-dynamic code sequence for a global `x`: + // + // Instruction Relocation Symbol + // ear %rX,%a0 + // sllg %rX,%rX,32 + // ear %rX,%a1 + // larl %r12,_GLOBAL_OFFSET_TABLE_ R_390_GOTPCDBL _GLOBAL_OFFSET_TABLE_ + // lgrl %r2,.LC0 R_390_PC32DBL .LC0 + // brasl %r14,__tls_get_offset@plt R_390_TLS_LDCALL + // :tls_ldcall: R_390_PLT32DBL __tls_get_offset + // la %r2,0(%r2,%rX) + // lgrl %rY,.LC1 R_390_PC32DBL .LC1 + // la %r2,0(%r2,%rY) + // + // .LC0: + // .quad @tlsldm R_390_TLS_LDM64 + // .LC1: + // .quad x@dtpoff R_390_TLS_LDO64 x + // + // Relaxing to local-exec entails: + // 1) Replacing the call by a nop. + // 2) Replacing the constant LC0 by 0 (i.e. ignoring the relocation). + // 3) Replacing the relocation on the constant LC1 by R_390_TLS_LE64. + + switch (rel.type) { + case R_390_TLS_LDCALL: + // brasl %r14,__tls_get_offset@plt -> brcl 0,. + write16be(loc, 0xc004); + write32be(loc + 2, 0x00000000); + break; + case R_390_TLS_LDM64: + break; + case R_390_TLS_LDO64: + relocateNoSym(loc, R_390_TLS_LE64, val); + break; + default: + llvm_unreachable("unsupported relocation for TLS LD to LE relaxation"); + } +} + +RelExpr SystemZ::adjustGotPcExpr(RelType type, int64_t addend, + const uint8_t *loc) const { + // Only R_390_GOTENT with addend 2 can be relaxed. + if (!config->relax || addend != 2 || type != R_390_GOTENT) + return R_GOT_PC; + const uint16_t op = read16be(loc - 2); + + // lgrl rx,sym@GOTENT -> larl rx, sym + // This relaxation is legal if "sym" binds locally (which was already + // verified by our caller) and is in-range and properly aligned for a + // LARL instruction. We cannot verify the latter constraint here, so + // we assume it is true and revert the decision later on in relaxOnce + // if necessary. + if ((op & 0xff0f) == 0xc408) + return R_RELAX_GOT_PC; + + return R_GOT_PC; +} + +bool SystemZ::relaxOnce(int pass) const { + // If we decided in adjustGotPcExpr to relax a R_390_GOTENT, + // we need to validate the target symbol is in-range and aligned. + SmallVector storage; + bool changed = false; + for (OutputSection *osec : outputSections) { + if (!(osec->flags & SHF_EXECINSTR)) + continue; + for (InputSection *sec : getInputSections(*osec, storage)) { + for (Relocation &rel : sec->relocs()) { + if (rel.expr != R_RELAX_GOT_PC) + continue; + + uint64_t v = sec->getRelocTargetVA( + sec->file, rel.type, rel.addend, + sec->getOutputSection()->addr + rel.offset, *rel.sym, rel.expr); + if (isInt<33>(v) && !(v & 1)) + continue; + if (rel.sym->auxIdx == 0) { + rel.sym->allocateAux(); + addGotEntry(*rel.sym); + changed = true; + } + rel.expr = R_GOT_PC; + } + } + } + return changed; +} + +void SystemZ::relaxGot(uint8_t *loc, const Relocation &rel, + uint64_t val) const { + assert(isInt<33>(val) && + "R_390_GOTENT should not have been relaxed if it overflows"); + assert(!(val & 1) && + "R_390_GOTENT should not have been relaxed if it is misaligned"); + const uint16_t op = read16be(loc - 2); + + // lgrl rx,sym@GOTENT -> larl rx, sym + if ((op & 0xff0f) == 0xc408) { + write16be(loc - 2, 0xc000 | (op & 0x00f0)); + write32be(loc, val >> 1); + } +} + +void SystemZ::relocate(uint8_t *loc, const Relocation &rel, + uint64_t val) const { + switch (rel.expr) { + case R_RELAX_GOT_PC: + return relaxGot(loc, rel, val); + case R_RELAX_TLS_GD_TO_IE_GOT_OFF: + return relaxTlsGdToIe(loc, rel, val); + case R_RELAX_TLS_GD_TO_LE: + return relaxTlsGdToLe(loc, rel, val); + case R_RELAX_TLS_LD_TO_LE: + return relaxTlsLdToLe(loc, rel, val); + default: + break; + } + switch (rel.type) { + case R_390_8: + checkIntUInt(loc, val, 8, rel); + *loc = val; + break; + case R_390_12: + case R_390_GOT12: + case R_390_GOTPLT12: + case R_390_TLS_GOTIE12: + checkUInt(loc, val, 12, rel); + write16be(loc, (read16be(loc) & 0xF000) | val); + break; + case R_390_PC12DBL: + case R_390_PLT12DBL: + checkInt(loc, val, 13, rel); + checkAlignment(loc, val, 2, rel); + write16be(loc, (read16be(loc) & 0xF000) | ((val >> 1) & 0x0FFF)); + break; + case R_390_16: + case R_390_GOT16: + case R_390_GOTPLT16: + case R_390_GOTOFF16: + case R_390_PLTOFF16: + checkIntUInt(loc, val, 16, rel); + write16be(loc, val); + break; + case R_390_PC16: + checkInt(loc, val, 16, rel); + write16be(loc, val); + break; + case R_390_PC16DBL: + case R_390_PLT16DBL: + checkInt(loc, val, 17, rel); + checkAlignment(loc, val, 2, rel); + write16be(loc, val >> 1); + break; + case R_390_20: + case R_390_GOT20: + case R_390_GOTPLT20: + case R_390_TLS_GOTIE20: + checkInt(loc, val, 20, rel); + write32be(loc, (read32be(loc) & 0xF00000FF) | ((val & 0xFFF) << 16) | + ((val & 0xFF000) >> 4)); + break; + case R_390_PC24DBL: + case R_390_PLT24DBL: + checkInt(loc, val, 25, rel); + checkAlignment(loc, val, 2, rel); + loc[0] = val >> 17; + loc[1] = val >> 9; + loc[2] = val >> 1; + break; + case R_390_32: + case R_390_GOT32: + case R_390_GOTPLT32: + case R_390_GOTOFF: + case R_390_PLTOFF32: + case R_390_TLS_IE32: + case R_390_TLS_GOTIE32: + case R_390_TLS_GD32: + case R_390_TLS_LDM32: + case R_390_TLS_LDO32: + case R_390_TLS_LE32: + checkIntUInt(loc, val, 32, rel); + write32be(loc, val); + break; + case R_390_PC32: + case R_390_PLT32: + checkInt(loc, val, 32, rel); + write32be(loc, val); + break; + case R_390_PC32DBL: + case R_390_PLT32DBL: + case R_390_GOTPCDBL: + case R_390_GOTENT: + case R_390_GOTPLTENT: + case R_390_TLS_IEENT: + checkInt(loc, val, 33, rel); + checkAlignment(loc, val, 2, rel); + write32be(loc, val >> 1); + break; + case R_390_64: + case R_390_PC64: + case R_390_PLT64: + case R_390_GOT64: + case R_390_GOTPLT64: + case R_390_GOTOFF64: + case R_390_PLTOFF64: + case R_390_GOTPC: + case R_390_TLS_IE64: + case R_390_TLS_GOTIE64: + case R_390_TLS_GD64: + case R_390_TLS_LDM64: + case R_390_TLS_LDO64: + case R_390_TLS_LE64: + case R_390_TLS_DTPMOD: + case R_390_TLS_DTPOFF: + case R_390_TLS_TPOFF: + write64be(loc, val); + break; + case R_390_TLS_LOAD: + case R_390_TLS_GDCALL: + case R_390_TLS_LDCALL: + break; + default: + llvm_unreachable("unknown relocation"); + } +} + +TargetInfo *elf::getSystemZTargetInfo() { + static SystemZ t; + return &t; +} diff --git a/lld/ELF/CMakeLists.txt b/lld/ELF/CMakeLists.txt index 475f7dea1dd7..83d816ddb060 100644 --- a/lld/ELF/CMakeLists.txt +++ b/lld/ELF/CMakeLists.txt @@ -33,6 +33,7 @@ add_lld_library(lldELF Arch/PPC64.cpp Arch/RISCV.cpp Arch/SPARCV9.cpp + Arch/SystemZ.cpp Arch/X86.cpp Arch/X86_64.cpp ARMErrataFix.cpp diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index c19fba6598a8..4bb9b7a0b2a9 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -200,6 +200,7 @@ static std::tuple parseEmulation(StringRef emul) { .Case("msp430elf", {ELF32LEKind, EM_MSP430}) .Case("elf64_amdgpu", {ELF64LEKind, EM_AMDGPU}) .Case("elf64loongarch", {ELF64LEKind, EM_LOONGARCH}) + .Case("elf64_s390", {ELF64BEKind, EM_S390}) .Default({ELFNoneKind, EM_NONE}); if (ret.first == ELFNoneKind) @@ -1137,7 +1138,7 @@ static SmallVector getSymbolOrderingFile(MemoryBufferRef mb) { static bool getIsRela(opt::InputArgList &args) { // The psABI specifies the default relocation entry format. bool rela = is_contained({EM_AARCH64, EM_AMDGPU, EM_HEXAGON, EM_LOONGARCH, - EM_PPC, EM_PPC64, EM_RISCV, EM_X86_64}, + EM_PPC, EM_PPC64, EM_RISCV, EM_S390, EM_X86_64}, config->emachine); // If -z rel or -z rela is specified, use the last option. for (auto *arg : args.filtered(OPT_z)) { diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index a292e873e72f..6c7ef27cbd49 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -1614,6 +1614,8 @@ static uint16_t getBitcodeMachineKind(StringRef path, const Triple &t) { return EM_RISCV; case Triple::sparcv9: return EM_SPARCV9; + case Triple::systemz: + return EM_S390; case Triple::x86: return t.isOSIAMCU() ? EM_IAMCU : EM_386; case Triple::x86_64: diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index 3d726b4c4b77..e033a715b592 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -655,6 +655,7 @@ static int64_t getTlsTpOffset(const Symbol &s) { // Variant 2. case EM_HEXAGON: + case EM_S390: case EM_SPARCV9: case EM_386: case EM_X86_64: @@ -717,6 +718,10 @@ uint64_t InputSectionBase::getRelocTargetVA(const InputFile *file, RelType type, case R_GOT_PC: case R_RELAX_TLS_GD_TO_IE: return sym.getGotVA() + a - p; + case R_GOTPLT_GOTREL: + return sym.getGotPltVA() + a - in.got->getVA(); + case R_GOTPLT_PC: + return sym.getGotPltVA() + a - p; case R_LOONGARCH_GOT_PAGE_PC: if (sym.hasFlag(NEEDS_TLSGD)) return getLoongArchPageDelta(in.got->getGlobalDynAddr(sym) + a, p, type); @@ -808,6 +813,8 @@ uint64_t InputSectionBase::getRelocTargetVA(const InputFile *file, RelType type, return getLoongArchPageDelta(sym.getPltVA() + a, p, type); case R_PLT_GOTPLT: return sym.getPltVA() + a - in.gotPlt->getVA(); + case R_PLT_GOTREL: + return sym.getPltVA() + a - in.got->getVA(); case R_PPC32_PLTREL: // R_PPC_PLTREL24 uses the addend (usually 0 or 0x8000) to indicate r30 // stores _GLOBAL_OFFSET_TABLE_ or .got2+0x8000. The addend is ignored for diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 79c8230724ad..f64b4219e0ac 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -203,8 +203,9 @@ static bool isAbsoluteValue(const Symbol &sym) { // Returns true if Expr refers a PLT entry. static bool needsPlt(RelExpr expr) { - return oneof(expr); + return oneof(expr); } bool lld::elf::needsGot(RelExpr expr) { @@ -233,6 +234,8 @@ static RelExpr toPlt(RelExpr expr) { return R_PLT_PC; case R_ABS: return R_PLT; + case R_GOTREL: + return R_PLT_GOTREL; default: return expr; } @@ -253,6 +256,8 @@ static RelExpr fromPlt(RelExpr expr) { return R_ABS; case R_PLT_GOTPLT: return R_GOTPLTREL; + case R_PLT_GOTREL: + return R_GOTREL; default: return expr; } @@ -979,10 +984,10 @@ bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type, if (oneof( - e)) + R_PLT_PC, R_PLT_GOTREL, R_PLT_GOTPLT, R_GOTPLT_GOTREL, R_GOTPLT_PC, + R_PPC32_PLTREL, R_PPC64_CALL_PLT, R_PPC64_RELAX_TOC, R_RISCV_ADD, + R_AARCH64_GOT_PAGE, R_LOONGARCH_PLT_PAGE_PC, R_LOONGARCH_GOT, + R_LOONGARCH_GOT_PAGE_PC>(e)) return true; // These never do, except if the entire file is position dependent or if @@ -1374,8 +1379,8 @@ static unsigned handleTlsRelocation(RelType type, Symbol &sym, R_LOONGARCH_GOT_PAGE_PC, R_GOT_OFF, R_TLSIE_HINT>(expr)) { ctx.hasTlsIe.store(true, std::memory_order_relaxed); // Initial-Exec relocs can be optimized to Local-Exec if the symbol is - // locally defined. - if (execOptimize && isLocalInExecutable) { + // locally defined. This is not supported on SystemZ. + if (execOptimize && isLocalInExecutable && config->emachine != EM_S390) { c.addReloc({R_RELAX_TLS_IE_TO_LE, type, offset, addend, &sym}); } else if (expr != R_TLSIE_HINT) { sym.setFlags(NEEDS_TLSIE); @@ -1534,8 +1539,10 @@ void RelocationScanner::scan(ArrayRef rels) { // For EhInputSection, OffsetGetter expects the relocations to be sorted by // r_offset. In rare cases (.eh_frame pieces are reordered by a linker // script), the relocations may be unordered. + // On SystemZ, all sections need to be sorted by r_offset, to allow TLS + // relaxation to be handled correctly - see SystemZ::getTlsGdRelaxSkip. SmallVector storage; - if (isa(sec)) + if (isa(sec) || config->emachine == EM_S390) rels = sortRels(rels, storage); end = static_cast(rels.end()); diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h index cfb9092149f3..7eb8a811e693 100644 --- a/lld/ELF/Relocations.h +++ b/lld/ELF/Relocations.h @@ -40,11 +40,14 @@ enum RelExpr { R_GOTPLT, R_GOTPLTREL, R_GOTREL, + R_GOTPLT_GOTREL, + R_GOTPLT_PC, R_NONE, R_PC, R_PLT, R_PLT_PC, R_PLT_GOTPLT, + R_PLT_GOTREL, R_RELAX_HINT, R_RELAX_GOT_PC, R_RELAX_GOT_PC_NOPIC, diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp index dd69916d6b05..f0ede1f43bbd 100644 --- a/lld/ELF/ScriptParser.cpp +++ b/lld/ELF/ScriptParser.cpp @@ -445,6 +445,7 @@ static std::pair parseBfdName(StringRef s) { .Case("elf32-msp430", {ELF32LEKind, EM_MSP430}) .Case("elf32-loongarch", {ELF32LEKind, EM_LOONGARCH}) .Case("elf64-loongarch", {ELF64LEKind, EM_LOONGARCH}) + .Case("elf64-s390", {ELF64BEKind, EM_S390}) .Default({ELFNoneKind, EM_NONE}); } diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 4b413163314b..bada394aa30d 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1419,6 +1419,9 @@ DynamicSection::computeContents() { case EM_MIPS: addInSec(DT_MIPS_PLTGOT, *in.gotPlt); break; + case EM_S390: + addInSec(DT_PLTGOT, *in.got); + break; case EM_SPARCV9: addInSec(DT_PLTGOT, *in.plt); break; diff --git a/lld/ELF/Target.cpp b/lld/ELF/Target.cpp index 286db1e3bbe0..d879a427e9c0 100644 --- a/lld/ELF/Target.cpp +++ b/lld/ELF/Target.cpp @@ -87,6 +87,8 @@ TargetInfo *elf::getTarget() { return getRISCVTargetInfo(); case EM_SPARCV9: return getSPARCV9TargetInfo(); + case EM_S390: + return getSystemZTargetInfo(); case EM_X86_64: return getX86_64TargetInfo(); default: diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h index ed00e81c0e6c..0cefa3181356 100644 --- a/lld/ELF/Target.h +++ b/lld/ELF/Target.h @@ -188,6 +188,7 @@ TargetInfo *getPPC64TargetInfo(); TargetInfo *getPPCTargetInfo(); TargetInfo *getRISCVTargetInfo(); TargetInfo *getSPARCV9TargetInfo(); +TargetInfo *getSystemZTargetInfo(); TargetInfo *getX86TargetInfo(); TargetInfo *getX86_64TargetInfo(); template TargetInfo *getMipsTargetInfo(); diff --git a/lld/test/ELF/Inputs/systemz-init.s b/lld/test/ELF/Inputs/systemz-init.s new file mode 100644 index 000000000000..1611b69b4419 --- /dev/null +++ b/lld/test/ELF/Inputs/systemz-init.s @@ -0,0 +1,5 @@ +// glibc < 2.39 used to align .init and .fini code at a 4-byte boundary. +// This file aims to recreate that behavior. + .section .init,"ax",@progbits + .align 4 + lg %r4, 272(%r15) diff --git a/lld/test/ELF/basic-systemz.s b/lld/test/ELF/basic-systemz.s new file mode 100644 index 000000000000..f7bb0e8cbd02 --- /dev/null +++ b/lld/test/ELF/basic-systemz.s @@ -0,0 +1,63 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld --hash-style=sysv -discard-all -shared %t.o -o %t.so +# RUN: llvm-readelf --file-header --program-headers --section-headers --dynamic-table %t.so | FileCheck %s + +# Exits with return code 55 on linux. +.text + lghi 2,55 + svc 1 + +# CHECK: ELF Header: +# CHECK-NEXT: Magic: 7f 45 4c 46 02 02 01 00 00 00 00 00 00 00 00 00 +# CHECK-NEXT: Class: ELF64 +# CHECK-NEXT: Data: 2's complement, big endian +# CHECK-NEXT: Version: 1 (current) +# CHECK-NEXT: OS/ABI: UNIX - System V +# CHECK-NEXT: ABI Version: 0 +# CHECK-NEXT: Type: DYN (Shared object file) +# CHECK-NEXT: Machine: IBM S/390 +# CHECK-NEXT: Version: 0x1 +# CHECK-NEXT: Entry point address: 0x0 +# CHECK-NEXT: Start of program headers: 64 (bytes into file) +# CHECK-NEXT: Start of section headers: 768 (bytes into file) +# CHECK-NEXT: Flags: 0x0 +# CHECK-NEXT: Size of this header: 64 (bytes) +# CHECK-NEXT: Size of program headers: 56 (bytes) +# CHECK-NEXT: Number of program headers: 7 +# CHECK-NEXT: Size of section headers: 64 (bytes) +# CHECK-NEXT: Number of section headers: 11 +# CHECK-NEXT: Section header string table index: 9 + +# CHECK: Section Headers: +# CHECK-NEXT: [Nr] Name Type Address Off Size ES Flg Lk Inf Al +# CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 +# CHECK-NEXT: [ 1] .dynsym DYNSYM 00000000000001c8 0001c8 000018 18 A 3 1 8 +# CHECK-NEXT: [ 2] .hash HASH 00000000000001e0 0001e0 000010 04 A 1 0 4 +# CHECK-NEXT: [ 3] .dynstr STRTAB 00000000000001f0 0001f0 000001 00 A 0 0 1 +# CHECK-NEXT: [ 4] .text PROGBITS 00000000000011f4 0001f4 000006 00 AX 0 0 4 +# CHECK-NEXT: [ 5] .dynamic DYNAMIC 0000000000002200 000200 000060 10 WA 3 0 8 +# CHECK-NEXT: [ 6] .relro_padding NOBITS 0000000000002260 000260 000da0 00 WA 0 0 1 +# CHECK-NEXT: [ 7] .comment PROGBITS 0000000000000000 000260 000008 01 MS 0 0 1 +# CHECK-NEXT: [ 8] .symtab SYMTAB 0000000000000000 000268 000030 18 10 2 8 +# CHECK-NEXT: [ 9] .shstrtab STRTAB 0000000000000000 000298 000058 00 0 0 1 +# CHECK-NEXT: [10] .strtab STRTAB 0000000000000000 0002f0 00000a 00 0 0 1 + +# CHECK: Program Headers: +# CHECK-NEXT: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align +# CHECK-NEXT: PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x000188 0x000188 R 0x8 +# CHECK-NEXT: LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x0001f1 0x0001f1 R 0x1000 +# CHECK-NEXT: LOAD 0x0001f4 0x00000000000011f4 0x00000000000011f4 0x000006 0x000006 R E 0x1000 +# CHECK-NEXT: LOAD 0x000200 0x0000000000002200 0x0000000000002200 0x000060 0x000e00 RW 0x1000 +# CHECK-NEXT: DYNAMIC 0x000200 0x0000000000002200 0x0000000000002200 0x000060 0x000060 RW 0x8 +# CHECK-NEXT: GNU_RELRO 0x000200 0x0000000000002200 0x0000000000002200 0x000060 0x000e00 R 0x1 +# CHECK-NEXT: GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW 0x0 + +# CHECK: Dynamic section at offset 0x200 contains 6 entries: +# CHECK-NEXT: Tag Type Name/Value +# CHECK-NEXT: 0x0000000000000006 (SYMTAB) 0x1c8 +# CHECK-NEXT: 0x000000000000000b (SYMENT) 24 (bytes) +# CHECK-NEXT: 0x0000000000000005 (STRTAB) 0x1f0 +# CHECK-NEXT: 0x000000000000000a (STRSZ) 1 (bytes) +# CHECK-NEXT: 0x0000000000000004 (HASH) 0x1e0 +# CHECK-NEXT: 0x0000000000000000 (NULL) 0x0 diff --git a/lld/test/ELF/emulation-systemz.s b/lld/test/ELF/emulation-systemz.s new file mode 100644 index 000000000000..dfdb4620954c --- /dev/null +++ b/lld/test/ELF/emulation-systemz.s @@ -0,0 +1,29 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld -m elf64_s390 %t.o -o %t1 +# RUN: llvm-readelf --file-header %t1 | FileCheck %s +# RUN: ld.lld %t.o -o %t2 +# RUN: llvm-readelf --file-header %t2 | FileCheck %s +# RUN: echo 'OUTPUT_FORMAT(elf64-s390)' > %t.script +# RUN: ld.lld %t.script %t.o -o %t3 +# RUN: llvm-readelf --file-header %t3 | FileCheck %s + +# CHECK: ELF Header: +# CHECK-NEXT: Magic: 7f 45 4c 46 02 02 01 00 00 00 00 00 00 00 00 00 +# CHECK-NEXT: Class: ELF64 +# CHECK-NEXT: Data: 2's complement, big endian +# CHECK-NEXT: Version: 1 (current) +# CHECK-NEXT: OS/ABI: UNIX - System V +# CHECK-NEXT: ABI Version: 0 +# CHECK-NEXT: Type: EXEC (Executable file) +# CHECK-NEXT: Machine: IBM S/390 +# CHECK-NEXT: Version: 0x1 +# CHECK-NEXT: Entry point address: +# CHECK-NEXT: Start of program headers: 64 (bytes into file) +# CHECK-NEXT: Start of section headers: +# CHECK-NEXT: Flags: 0x0 +# CHECK-NEXT: Size of this header: 64 (bytes) +# CHECK-NEXT: Size of program headers: 56 (bytes) + +.globl _start +_start: diff --git a/lld/test/ELF/lto/systemz.ll b/lld/test/ELF/lto/systemz.ll new file mode 100644 index 000000000000..42bf4e32fb6d --- /dev/null +++ b/lld/test/ELF/lto/systemz.ll @@ -0,0 +1,18 @@ +; REQUIRES: systemz +;; Test we can infer the e_machine value EM_S390 from a bitcode file. + +; RUN: llvm-as %s -o %t.o +; RUN: ld.lld %t.o -o %t +; RUN: llvm-readobj -h %t | FileCheck %s + +; CHECK: Class: 64-bit +; CHECK: DataEncoding: BigEndian +; CHECK: Machine: EM_S390 + +target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64" +target triple = "s390x-unknown-linux-gnu" + +define void @_start() { +entry: + ret void +} diff --git a/lld/test/ELF/systemz-got.s b/lld/test/ELF/systemz-got.s new file mode 100644 index 000000000000..1d558aa3b029 --- /dev/null +++ b/lld/test/ELF/systemz-got.s @@ -0,0 +1,16 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %p/Inputs/shared.s -o %t2.o +# RUN: ld.lld -shared %t2.o -soname=%t2.so -o %t2.so + +# RUN: ld.lld -dynamic-linker /lib/ld64.so.1 %t.o %t2.so -o %t +# RUN: llvm-readelf -S -r %t | FileCheck %s + +# CHECK: .got PROGBITS {{.*}} {{.*}} 000020 00 WA 0 0 8 + +# CHECK: Relocation section '.rela.dyn' at offset {{.*}} contains 1 entries: +# CHECK: {{.*}} 000000010000000a R_390_GLOB_DAT 0000000000000000 bar + 0 + +.global _start +_start: + lgrl %r1,bar@GOT diff --git a/lld/test/ELF/systemz-gotent-relax-align.s b/lld/test/ELF/systemz-gotent-relax-align.s new file mode 100644 index 000000000000..c6326086f56d --- /dev/null +++ b/lld/test/ELF/systemz-gotent-relax-align.s @@ -0,0 +1,48 @@ +# REQUIRES: systemz +## Verify that R_390_GOTENT optimization is not performed on misaligned symbols. + +# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld %t.o -o %t1 +# RUN: llvm-readelf -S -r -x .got -x .got.plt %t1 | FileCheck --check-prefixes=CHECK %s +# RUN: llvm-objdump --no-print-imm-hex -d %t1 | FileCheck --check-prefix=DISASM %s + +## We retain one .got entry for the unaligned symbol. +# CHECK: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK: .got PROGBITS 00000000010021e0 0001e0 000020 00 WA 0 0 8 +# CHECK-NEXT: .relro_padding NOBITS 0000000001002200 000200 000e00 00 WA 0 0 1 +# CHECK-NEXT: .data PROGBITS 0000000001003200 000200 000006 00 WA 0 0 2 + +# CHECK-LABEL: Hex dump of section '.got': +# CHECK-NEXT: 0x010021e0 00000000 00000000 00000000 00000000 +# CHECK-NEXT: 0x010021f0 00000000 00000000 00000000 01003205 + +# DISASM: Disassembly of section .text: +# DISASM: <_start>: +# DISASM-NEXT: larl %r1, 0x1003200 +# DISASM-NEXT: larl %r1, 0x1003200 +# DISASM-NEXT: lgrl %r1, 0x10021f8 +# DISASM-NEXT: lgrl %r1, 0x10021f8 + +.data +.globl var_align +.hidden var_align + .align 2 +var_align: + .long 0 + +.data +.globl var_unalign +.hidden var_unalign + .align 2 + .byte 0 +var_unalign: + .byte 0 + +.text +.globl _start +.type _start, @function +_start: + lgrl %r1, var_align@GOT + lgrl %r1, var_align@GOT + lgrl %r1, var_unalign@GOT + lgrl %r1, var_unalign@GOT diff --git a/lld/test/ELF/systemz-gotent-relax-und-dso.s b/lld/test/ELF/systemz-gotent-relax-und-dso.s new file mode 100644 index 000000000000..57369a417fd4 --- /dev/null +++ b/lld/test/ELF/systemz-gotent-relax-und-dso.s @@ -0,0 +1,68 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %S/Inputs/gotpc-relax-und-dso.s -o %tdso.o +# RUN: ld.lld -shared %tdso.o -soname=t.so -o %t.so +# RUN: ld.lld --hash-style=sysv -shared %t.o %t.so -o %t +# RUN: llvm-readelf -r %t | FileCheck --check-prefix=RELOC %s +# RUN: llvm-objdump --no-print-imm-hex -d %t | FileCheck --check-prefix=DISASM %s + +# RELOC-LABEL: Relocation section '.rela.dyn' at offset {{.*}} contains 3 entries: +# RELOC: 00000000000023f8 000000010000000a R_390_GLOB_DAT 00000000000012d8 foo + 0 +# RELOC: 0000000000002400 000000030000000a R_390_GLOB_DAT 0000000000000000 und + 0 +# RELOC: 0000000000002408 000000040000000a R_390_GLOB_DAT 0000000000000000 dsofoo + 0 + +# DISASM: Disassembly of section .text: +# DISASM-EMPTY: +# DISASM-NEXT: : +# DISASM-NEXT: bc 0, 0 +# DISASM: : +# DISASM-NEXT: bc 0, 0 +# DISASM: <_start>: +# DISASM-NEXT: lgrl %r1, 0x2400 +# DISASM-NEXT: lgrl %r1, 0x2400 +# DISASM-NEXT: lgrl %r1, 0x2408 +# DISASM-NEXT: lgrl %r1, 0x2408 +# DISASM-NEXT: larl %r1, 0x12dc +# DISASM-NEXT: larl %r1, 0x12dc +# DISASM-NEXT: lgrl %r1, 0x23f8 +# DISASM-NEXT: lgrl %r1, 0x23f8 +# DISASM-NEXT: lgrl %r1, 0x2400 +# DISASM-NEXT: lgrl %r1, 0x2400 +# DISASM-NEXT: lgrl %r1, 0x2408 +# DISASM-NEXT: lgrl %r1, 0x2408 +# DISASM-NEXT: larl %r1, 0x12dc +# DISASM-NEXT: larl %r1, 0x12dc +# DISASM-NEXT: lgrl %r1, 0x23f8 +# DISASM-NEXT: lgrl %r1, 0x23f8 + +.text +.globl foo +.type foo, @function +foo: + nop + +.globl hid +.hidden hid +.type hid, @function +hid: + nop + +.globl _start +.type _start, @function +_start: + lgrl %r1, und@GOT + lgrl %r1, und@GOT + lgrl %r1, dsofoo@GOT + lgrl %r1, dsofoo@GOT + lgrl %r1, hid@GOT + lgrl %r1, hid@GOT + lgrl %r1, foo@GOT + lgrl %r1, foo@GOT + lgrl %r1, und@GOT + lgrl %r1, und@GOT + lgrl %r1, dsofoo@GOT + lgrl %r1, dsofoo@GOT + lgrl %r1, hid@GOT + lgrl %r1, hid@GOT + lgrl %r1, foo@GOT + lgrl %r1, foo@GOT diff --git a/lld/test/ELF/systemz-gotent-relax.s b/lld/test/ELF/systemz-gotent-relax.s new file mode 100644 index 000000000000..f665e1af9e53 --- /dev/null +++ b/lld/test/ELF/systemz-gotent-relax.s @@ -0,0 +1,91 @@ +# REQUIRES: systemz +## Test R_390_GOTENT optimization. + +# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld %t.o -o %t1 --no-apply-dynamic-relocs +# RUN: llvm-readelf -S -r -x .got.plt %t1 | FileCheck --check-prefixes=CHECK,NOAPPLY %s +# RUN: ld.lld %t.o -o %t1 --apply-dynamic-relocs +# RUN: llvm-readelf -S -r -x .got.plt %t1 | FileCheck --check-prefixes=CHECK,APPLY %s +# RUN: ld.lld %t.o -o %t1 +# RUN: llvm-objdump --no-print-imm-hex -d %t1 | FileCheck --check-prefix=DISASM %s + +## --no-relax disables GOT optimization. +# RUN: ld.lld --no-relax %t.o -o %t2 +# RUN: llvm-objdump --no-print-imm-hex -d %t2 | FileCheck --check-prefix=NORELAX %s + +## In our implementation, .got is retained even if all GOT-generating relocations are optimized. +# CHECK: Name Type Address Off Size ES Flg Lk Inf Al +# CHECK: .iplt PROGBITS 0000000001001240 000240 000020 00 AX 0 0 16 +# CHECK-NEXT: .got PROGBITS 0000000001002260 000260 000018 00 WA 0 0 8 +# CHECK-NEXT: .relro_padding NOBITS 0000000001002278 000278 000d88 00 WA 0 0 1 +# CHECK-NEXT: .got.plt PROGBITS 0000000001003278 000278 000008 00 WA 0 0 8 + +## There is one R_S390_IRELATIVE relocation. +# CHECK-LABEL: Relocation section '.rela.dyn' at offset {{.*}} contains 1 entries: +# CHECK: 0000000001003278 000000000000003d R_390_IRELATIVE 10011e8 + +# CHECK-LABEL: Hex dump of section '.got.plt': +# NOAPPLY-NEXT: 0x01003278 00000000 00000000 +# APPLY-NEXT: 0x01003278 00000000 010011e8 + +# DISASM: Disassembly of section .text: +# DISASM: 00000000010011e0 : +# DISASM-NEXT: bc 0, 0 +# DISASM: 00000000010011e4 : +# DISASM-NEXT: bc 0, 0 +# DISASM: 00000000010011e8 : +# DISASM-NEXT: br %r14 +# DISASM: 00000000010011ea <_start>: +# DISASM-NEXT: larl %r1, 0x10011e0 +# DISASM-NEXT: larl %r1, 0x10011e0 +# DISASM-NEXT: larl %r1, 0x10011e4 +# DISASM-NEXT: larl %r1, 0x10011e4 +# DISASM-NEXT: lgrl %r1, 0x1003278 +# DISASM-NEXT: lgrl %r1, 0x1003278 +# DISASM-NEXT: larl %r1, 0x10011e0 +# DISASM-NEXT: larl %r1, 0x10011e0 +# DISASM-NEXT: larl %r1, 0x10011e4 +# DISASM-NEXT: larl %r1, 0x10011e4 +# DISASM-NEXT: lgrl %r1, 0x1003278 +# DISASM-NEXT: lgrl %r1, 0x1003278 + +# NORELAX-LABEL: <_start>: +# NORELAX-COUNT-12: lgrl + +.text +.globl foo + +.text +.globl foo +.type foo, @function +foo: + nop + +.globl hid +.hidden hid +.type hid, @function +hid: + nop + +.text +.type ifunc STT_GNU_IFUNC +.globl ifunc +.type ifunc, @function +ifunc: + br %r14 + +.globl _start +.type _start, @function +_start: + lgrl %r1, foo@GOT + lgrl %r1, foo@GOT + lgrl %r1, hid@GOT + lgrl %r1, hid@GOT + lgrl %r1, ifunc@GOT + lgrl %r1, ifunc@GOT + lgrl %r1, foo@GOT + lgrl %r1, foo@GOT + lgrl %r1, hid@GOT + lgrl %r1, hid@GOT + lgrl %r1, ifunc@GOT + lgrl %r1, ifunc@GOT diff --git a/lld/test/ELF/systemz-ifunc-nonpreemptible.s b/lld/test/ELF/systemz-ifunc-nonpreemptible.s new file mode 100644 index 000000000000..5056db302ca1 --- /dev/null +++ b/lld/test/ELF/systemz-ifunc-nonpreemptible.s @@ -0,0 +1,75 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-none-linux-gnu %s -o %t.o +# RUN: ld.lld -static %t.o -o %t +# RUN: ld.lld -static %t.o -o %t.apply --apply-dynamic-relocs +# RUN: llvm-readelf --section-headers --relocations --symbols %t | FileCheck %s +# RUN: llvm-readelf -x .got.plt %t | FileCheck %s --check-prefix=NO-APPLY-RELOC +# RUN: llvm-readelf -x .got.plt %t.apply | FileCheck %s --check-prefix=APPLY-RELOC +# RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t | FileCheck %s --check-prefix=DISASM + +# CHECK: Section Headers: +# CHECK-NEXT: [Nr] Name Type Address Off Size ES Flg Lk Inf Al +# CHECK-NEXT: [ 0] NULL 0000000000000000 000000 000000 00 0 0 0 +# CHECK-NEXT: [ 1] .rela.dyn RELA 0000000001000158 000158 000030 18 AI 0 4 8 +# CHECK-NEXT: [ 2] .text PROGBITS 0000000001001188 000188 00001c 00 AX 0 0 4 +# CHECK-NEXT: [ 3] .iplt PROGBITS 00000000010011b0 0001b0 000040 00 AX 0 0 16 +# CHECK-NEXT: [ 4] .got.plt PROGBITS 00000000010021f0 0001f0 000010 00 WA 0 0 8 + +# CHECK: Relocation section '.rela.dyn' at offset 0x158 contains 2 entries: +# CHECK-NEXT: Offset Info Type Symbol's Value Symbol's Name + Addend +# CHECK-NEXT: 00000000010021f0 000000000000003d R_390_IRELATIVE 1001188 +# CHECK-NEXT: 00000000010021f8 000000000000003d R_390_IRELATIVE 100118a + +# CHECK: Symbol table '.symtab' contains 6 entries: +# CHECK-NEXT: Num: Value Size Type Bind Vis Ndx Name +# CHECK-NEXT: 0: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND +# CHECK-NEXT: 1: 0000000001000158 0 NOTYPE LOCAL HIDDEN 1 __rela_iplt_start +# CHECK-NEXT: 2: 0000000001000188 0 NOTYPE LOCAL HIDDEN 1 __rela_iplt_end +# CHECK-NEXT: 3: 0000000001001188 0 IFUNC GLOBAL DEFAULT 2 foo +# CHECK-NEXT: 4: 000000000100118a 0 IFUNC GLOBAL DEFAULT 2 bar +# CHECK-NEXT: 5: 000000000100118c 0 NOTYPE GLOBAL DEFAULT 2 _start + +# NO-APPLY-RELOC-LABEL: Hex dump of section '.got.plt': +# NO-APPLY-RELOC-NEXT: 0x010021f0 00000000 00000000 00000000 00000000 +# NO-APPLY-RELOC-EMPTY: + +# APPLY-RELOC-LABEL: Hex dump of section '.got.plt': +# APPLY-RELOC-NEXT: 0x010021f0 00000000 01001188 00000000 0100118a +# APPLY-RELOC-EMPTY: + +# DISASM: Disassembly of section .text: +# DISASM: 0000000001001188 : +# DISASM-NEXT: br %r14 +# DISASM: 000000000100118a : +# DISASM-NEXT: br %r14 +# DISASM: 000000000100118c <_start>: +# DISASM-NEXT: brasl %r14, 0x10011b0 +# DISASM-NEXT: brasl %r14, 0x10011d0 +# DISASM-NEXT: larl %r2, 0x1000158 +# DISASM-NEXT: larl %r2, 0x1000188 +# DISASM: Disassembly of section .iplt: +# DISASM: <.iplt>: +# DISASM: 10011b0: larl %r1, 0x10021f0 +# DISASM-NEXT: 10011b6: lg %r1, 0(%r1) +# DISASM-NEXT: 10011bc: br %r1 +# DISASM: 10011d0: larl %r1, 0x10021f8 +# DISASM-NEXT: 10011d6: lg %r1, 0(%r1) +# DISASM-NEXT: 10011dc: br %r1 + +.text +.type foo STT_GNU_IFUNC +.globl foo +foo: + br %r14 + +.type bar STT_GNU_IFUNC +.globl bar +bar: + br %r14 + +.globl _start +_start: + brasl %r14, foo@plt + brasl %r14, bar@plt + larl %r2, __rela_iplt_start + larl %r2, __rela_iplt_end diff --git a/lld/test/ELF/systemz-init-padding.s b/lld/test/ELF/systemz-init-padding.s new file mode 100644 index 000000000000..c56b98d43f1b --- /dev/null +++ b/lld/test/ELF/systemz-init-padding.s @@ -0,0 +1,27 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %p/Inputs/systemz-init.s -o systemz-init.o +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld -dynamic-linker /lib/ld64.so.1 %t.o systemz-init.o -o %t +# RUN: llvm-objdump -d --no-show-raw-insn -j .init %t | FileCheck %s + +# glibc < 2.39 used to align .init and .fini code at a 4-byte boundary. +# When that happens, the linker must not pad the code with invalid +# instructions, e.g. null bytes. + .section .init,"ax",@progbits + brasl %r14, startup + +# CHECK: <.init>: +# CHECK-NEXT: brasl %r14, +# CHECK-NEXT: bcr 0, %r7 +# CHECK-NEXT: lg %r4, 272(%r15) + + .text + .globl startup + .p2align 4 +startup: + br %r14 + + .globl main + .p2align 4 +main: + br %r14 diff --git a/lld/test/ELF/systemz-pie.s b/lld/test/ELF/systemz-pie.s new file mode 100644 index 000000000000..bb971a82fb8c --- /dev/null +++ b/lld/test/ELF/systemz-pie.s @@ -0,0 +1,38 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t1.o + +## Check -pie. +# RUN: ld.lld -pie %t1.o -o %t +# RUN: llvm-readelf --file-headers --program-headers --dynamic %t | FileCheck %s + +# CHECK: ELF Header: +# CHECK-NEXT: Magic: 7f 45 4c 46 02 02 01 00 00 00 00 00 00 00 00 00 +# CHECK-NEXT: Class: ELF64 +# CHECK-NEXT: Data: 2's complement, big endian +# CHECK-NEXT: Version: 1 (current) +# CHECK-NEXT: OS/ABI: UNIX - System V +# CHECK-NEXT: ABI Version: 0 +# CHECK-NEXT: Type: DYN (Shared object file) +# CHECK-NEXT: Machine: IBM S/390 +# CHECK-NEXT: Version: 0x1 + +# CHECK: Program Headers: +# CHECK-NEXT: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align +# CHECK-NEXT: PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x000188 0x000188 R 0x8 +# CHECK-NEXT: LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x00020d 0x00020d R 0x1000 +# CHECK-NEXT: LOAD 0x000210 0x0000000000002210 0x0000000000002210 0x000090 0x000df0 RW 0x1000 +# CHECK-NEXT: DYNAMIC 0x000210 0x0000000000002210 0x0000000000002210 0x000090 0x000090 RW 0x8 +# CHECK-NEXT: GNU_RELRO 0x000210 0x0000000000002210 0x0000000000002210 0x000090 0x000df0 R 0x1 +# CHECK-NEXT: GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW 0x0 + +# CHECK: Dynamic section at offset 0x210 contains 9 entries: +# CHECK-NEXT: Tag Type Name/Value +# CHECK-NEXT: 0x000000006ffffffb (FLAGS_1) PIE + +## Check -nopie +# RUN: ld.lld -no-pie %t1.o -o %t2 +# RUN: llvm-readelf --file-headers %t2 | FileCheck %s --check-prefix=NOPIE +# NOPIE-NOT: Type: DYN + +.globl _start +_start: diff --git a/lld/test/ELF/systemz-plt.s b/lld/test/ELF/systemz-plt.s new file mode 100644 index 000000000000..4669f01f5881 --- /dev/null +++ b/lld/test/ELF/systemz-plt.s @@ -0,0 +1,83 @@ +# REQUIRES: systemz +# RUN: echo '.globl bar, weak; .type bar,@function; .type weak,@function; bar: weak:' > %t1.s + +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %t1.s -o %t1.o +# RUN: ld.lld -shared %t1.o -soname=t1.so -o %t1.so +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld %t.o %t1.so -z separate-code -o %t +# RUN: llvm-readelf -S -s -r -x .got.plt %t | FileCheck %s +# RUN: llvm-objdump -d %t | FileCheck --check-prefixes=DIS %s + +# CHECK: Section Headers: +# CHECK: .plt PROGBITS 0000000001001020 001020 000060 00 AX 0 0 16 +# CHECK: .got PROGBITS 00000000010020d0 0020d0 000018 00 WA 0 0 8 +# CHECK: .got.plt PROGBITS 00000000010030e8 0020e8 000010 00 WA 0 0 8 + +# CHECK: Relocation section '.rela.plt' at offset {{.*}} contains 2 entries: +# CHECK: 00000000010030e8 000000010000000b R_390_JMP_SLOT 0000000000000000 bar + 0 +# CHECK: 00000000010030f0 000000020000000b R_390_JMP_SLOT 0000000000000000 weak + 0 + +## A canonical PLT has a non-zero st_value. bar and weak are called but their +## addresses are not taken, so a canonical PLT is not necessary. +# CHECK: Symbol table '.dynsym' contains 3 entries: +# CHECK-NEXT: Num: Value Size Type Bind Vis Ndx Name +# CHECK-NEXT: 0: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND +# CHECK-NEXT: 1: 0000000000000000 0 FUNC GLOBAL DEFAULT UND bar +# CHECK-NEXT: 2: 0000000000000000 0 FUNC WEAK DEFAULT UND weak + +## The .got.plt slots relocated by .rela.plt point to .plt +## This is required by glibc. +# CHECK: Hex dump of section '.got.plt': +# CHECK-NEXT: 0x010030e8 00000000 0100104e 00000000 0100106e + +# DIS: Disassembly of section .text: + +# DIS: 0000000001001000 <_start>: +# DIS-NEXT: brasl %r14, 0x1001012 +# DIS-NEXT: brasl %r14, 0x1001040 +# DIS-NEXT: brasl %r14, 0x1001060 + +# DIS: 0000000001001012 : +# DIS-NEXT: br %r14 + +# DIS: Disassembly of section .plt: + +# DIS: 0000000001001020 <.plt>: +# DIS-NEXT: 1001020: e3 10 f0 38 00 24 stg %r1, 56(%r15) +# DIS-NEXT: 1001026: c0 10 00 00 08 55 larl %r1, 0x10020d0 +# DIS-NEXT: 100102c: d2 07 f0 30 10 08 mvc 48(8,%r15), 8(%r1) +# DIS-NEXT: 1001032: e3 10 10 10 00 04 lg %r1, 16(%r1) +# DIS-NEXT: 1001038: 07 f1 br %r1 +# DIS-NEXT: 100103a: 07 00 bcr 0, %r0 +# DIS-NEXT: 100103c: 07 00 bcr 0, %r0 +# DIS-NEXT: 100103e: 07 00 bcr 0, %r0 +# DIS-NEXT: 1001040: c0 10 00 00 10 54 larl %r1, 0x10030e8 +# DIS-NEXT: 1001046: e3 10 10 00 00 04 lg %r1, 0(%r1) +# DIS-NEXT: 100104c: 07 f1 br %r1 +# DIS-NEXT: 100104e: 0d 10 basr %r1, 0 +# DIS-NEXT: 1001050: e3 10 10 0c 00 14 lgf %r1, 12(%r1) +# DIS-NEXT: 1001056: c0 f4 ff ff ff e5 jg 0x1001020 +# DIS-NEXT: 100105c: 00 00 +# DIS-NEXT: 100105e: 00 00 +# DIS-NEXT: 1001060: c0 10 00 00 10 48 larl %r1, 0x10030f0 +# DIS-NEXT: 1001066: e3 10 10 00 00 04 lg %r1, 0(%r1) +# DIS-NEXT: 100106c: 07 f1 br %r1 +# DIS-NEXT: 100106e: 0d 10 basr %r1, 0 +# DIS-NEXT: 1001070: e3 10 10 0c 00 14 lgf %r1, 12(%r1) +# DIS-NEXT: 1001076: c0 f4 ff ff ff d5 jg 0x1001020 +# DIS-NEXT: 100107c: 00 00 +# DIS-NEXT: 100107e: 00 18 + +.global _start, foo, bar +.weak weak + +_start: + ## Use @plt to avoid generating direct references that would force + ## allocation of a canonical PLT entry. + brasl %r14, foo@plt + brasl %r14, bar@plt + brasl %r14, weak@plt + +## foo is local and non-preemptable, no PLT is generated. +foo: + br %r14 diff --git a/lld/test/ELF/systemz-reloc-abs.s b/lld/test/ELF/systemz-reloc-abs.s new file mode 100644 index 000000000000..b5ad94d90d3a --- /dev/null +++ b/lld/test/ELF/systemz-reloc-abs.s @@ -0,0 +1,32 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=s390x %S/Inputs/abs255.s -o %t255.o +# RUN: llvm-mc -filetype=obj -triple=s390x %S/Inputs/abs256.s -o %t256.o +# RUN: llvm-mc -filetype=obj -triple=s390x %S/Inputs/abs257.s -o %t257.o + +# RUN: ld.lld %t.o %t256.o -o %t +# RUN: llvm-readelf -x .data %t | FileCheck %s +# CHECK: 0x{{[0-9a-f]+}} ff80ffff 8000ffff ffff8000 0000ffff +# CHECK-NEXT: ffffffff ffff8000 00000000 0000 + +# RUN: not ld.lld %t.o %t255.o -o /dev/null 2>&1 | FileCheck --check-prefix=OVERFLOW1 %s +# OVERFLOW1: relocation R_390_8 out of range: -129 is not in [-128, 255] +# OVERFLOW1: relocation R_390_16 out of range: -32769 is not in [-32768, 65535] +# OVERFLOW1: relocation R_390_32 out of range: -2147483649 is not in [-2147483648, 4294967295] + +# RUN: not ld.lld %t.o %t257.o -o /dev/null 2>&1 | FileCheck --check-prefix=OVERFLOW2 %s +# OVERFLOW2: relocation R_390_8 out of range: 256 is not in [-128, 255] +# OVERFLOW2: relocation R_390_16 out of range: 65536 is not in [-32768, 65535] +# OVERFLOW2: relocation R_390_32 out of range: 4294967296 is not in [-2147483648, 4294967295] + +.globl _start +_start: +.data +.byte foo - 1 +.byte foo - 384 +.word foo + 0xfeff +.word foo - 0x8100 +.long foo + 0xfffffeff +.long foo - 0x80000100 +.quad foo + 0xfffffffffffffeff +.quad foo - 0x8000000000000100 diff --git a/lld/test/ELF/systemz-reloc-disp12.s b/lld/test/ELF/systemz-reloc-disp12.s new file mode 100644 index 000000000000..3d32707d149f --- /dev/null +++ b/lld/test/ELF/systemz-reloc-disp12.s @@ -0,0 +1,21 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym DISP=291 %s -o %t1.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym DISP=4095 %s -o %t2.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym DISP=4096 %s -o %t3.o + +# RUN: ld.lld --section-start=.text=0x0 %t1.o -o %t1out +# RUN: ld.lld --section-start=.text=0x0 %t2.o -o %t2out +# RUN: not ld.lld --section-start=.text=0x0 %t3.o -o /dev/null 2>&1 | FileCheck %s --check-prefix RANGE + +# RANGE: relocation R_390_12 out of range: 4096 is not in [0, 4095] + +# RUN: llvm-readelf --hex-dump=.text %t1out | FileCheck %s -DINSN=58678123 --check-prefix DUMP +# RUN: llvm-readelf --hex-dump=.text %t2out | FileCheck %s -DINSN=58678fff --check-prefix DUMP + +# DUMP: 0x00000000 [[INSN]] + +.text +.globl _start +_start: + .reloc .+2, R_390_12, DISP + l %r6, 0(%r7,%r8) diff --git a/lld/test/ELF/systemz-reloc-disp20.s b/lld/test/ELF/systemz-reloc-disp20.s new file mode 100644 index 000000000000..88cd657c6ae3 --- /dev/null +++ b/lld/test/ELF/systemz-reloc-disp20.s @@ -0,0 +1,21 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym DISP=74565 %s -o %t1.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym DISP=524287 %s -o %t2.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym DISP=524288 %s -o %t3.o + +# RUN: ld.lld --section-start=.text=0x0 %t1.o -o %t1out +# RUN: ld.lld --section-start=.text=0x0 %t2.o -o %t2out +# RUN: not ld.lld --section-start=.text=0x0 %t3.o -o /dev/null 2>&1 | FileCheck %s --check-prefix RANGE + +# RANGE: relocation R_390_20 out of range: 524288 is not in [-524288, 524287] + +# RUN: llvm-readelf --hex-dump=.text %t1out | FileCheck %s -DINSN="e3678345 1204" --check-prefix DUMP +# RUN: llvm-readelf --hex-dump=.text %t2out | FileCheck %s -DINSN="e3678fff 7f04" --check-prefix DUMP + +# DUMP: 0x00000000 [[INSN]] + +.text +.globl _start +_start: + .reloc .+2, R_390_20, DISP + lg %r6, 0(%r7,%r8) diff --git a/lld/test/ELF/systemz-reloc-got.s b/lld/test/ELF/systemz-reloc-got.s new file mode 100644 index 000000000000..4b9ac16481f4 --- /dev/null +++ b/lld/test/ELF/systemz-reloc-got.s @@ -0,0 +1,92 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld -z norelro -shared %t.o -soname=t.so -o %t.so +## Note: Without norelro the distance between .got and .got.plt causes +## R_390_GOTPLT12 relocations to always overflow. + +# RUN: llvm-readelf -S -x .data %t.so | FileCheck %s +# RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck %s --check-prefix=DISASM + +# CHECK: Section Headers: +# CHECK: .got PROGBITS 0000000000002458 +# CHECK: .got.plt PROGBITS 0000000000002480 + +## Note: _GLOBAL_OFFSET_TABLE is at .got +## GOT (foo) is at .got + 24 == 0x2470 +## GOT (bar) is at .got + 32 == 0x2478 +## GOTPLT (foo) is at .got.plt + 0 == .got + 40 == 0x2480 +## GOTPLT (bar) is at .got.plt + 8 == .got + 48 == 0x2488 + +# DISASM: larl %r12, 0x2458 +# DISASM-NEXT: larl %r1, 0x2470 +# DISASM-NEXT: larl %r1, 0x2478 +# DISASM-NEXT: larl %r1, 0x2480 +# DISASM-NEXT: larl %r1, 0x2488 + +# DISASM-NEXT: l %r1, 24(%r12) +# DISASM-NEXT: l %r1, 32(%r12) +# DISASM-NEXT: l %r1, 40(%r12) +# DISASM-NEXT: l %r1, 48(%r12) +# DISASM-NEXT: lg %r1, 24(%r12) +# DISASM-NEXT: lg %r1, 32(%r12) +# DISASM-NEXT: lg %r1, 40(%r12) +# DISASM-NEXT: lg %r1, 48(%r12) + +# CHECK: Hex dump of section '.data': +# CHECK-NEXT: 00180020 00280030 00000018 00000020 +# CHECK-NEXT: 00000028 00000030 00000000 00000018 +# CHECK-NEXT: 00000000 00000020 00000000 00000028 +# CHECK-NEXT: 00000000 00000030 + +.text +larl %r12, _GLOBAL_OFFSET_TABLE_ +.reloc .+2, R_390_GOTENT, foo+2 +larl %r1, 0 +.reloc .+2, R_390_GOTENT, bar+2 +larl %r1, 0 +.reloc .+2, R_390_GOTPLTENT, foo+2 +larl %r1, 0 +.reloc .+2, R_390_GOTPLTENT, bar+2 +larl %r1, 0 +.reloc .+2, R_390_GOT12, foo +l %r1, 0(%r12) +.reloc .+2, R_390_GOT12, bar +l %r1, 0(%r12) +.reloc .+2, R_390_GOTPLT12, foo +l %r1, 0(%r12) +.reloc .+2, R_390_GOTPLT12, bar +l %r1, 0(%r12) +.reloc .+2, R_390_GOT20, foo +lg %r1, 0(%r12) +.reloc .+2, R_390_GOT20, bar +lg %r1, 0(%r12) +.reloc .+2, R_390_GOTPLT20, foo +lg %r1, 0(%r12) +.reloc .+2, R_390_GOTPLT20, bar +lg %r1, 0(%r12) + +.data +.reloc ., R_390_GOT16, foo +.space 2 +.reloc ., R_390_GOT16, bar +.space 2 +.reloc ., R_390_GOTPLT16, foo +.space 2 +.reloc ., R_390_GOTPLT16, bar +.space 2 +.reloc ., R_390_GOT32, foo +.space 4 +.reloc ., R_390_GOT32, bar +.space 4 +.reloc ., R_390_GOTPLT32, foo +.space 4 +.reloc ., R_390_GOTPLT32, bar +.space 4 +.reloc ., R_390_GOT64, foo +.space 8 +.reloc ., R_390_GOT64, bar +.space 8 +.reloc ., R_390_GOTPLT64, foo +.space 8 +.reloc ., R_390_GOTPLT64, bar +.space 8 diff --git a/lld/test/ELF/systemz-reloc-gotrel.s b/lld/test/ELF/systemz-reloc-gotrel.s new file mode 100644 index 000000000000..46669ecfa7fd --- /dev/null +++ b/lld/test/ELF/systemz-reloc-gotrel.s @@ -0,0 +1,36 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: ld.lld -shared %t.o -soname=t.so -o %t.so + +# RUN: llvm-readelf -S -s -x .data %t.so | FileCheck %s + +# CHECK: Section Headers: +# CHECK: .plt PROGBITS 0000000000001290 +# CHECK: .got PROGBITS 0000000000002390 + +# CHECK: Symbol table '.symtab' +# CHECK: 0000000000001288 {{.*}} bar + +## Note: foo is the first (and only) PLT entry, which resides at .plt + 32 +## PLTOFF (foo) is (.plt + 32) - .got == 0x12b0 - 0x2390 == 0xffffef20 +## GOTOFF (bar) is bar - .got == 0x1288 - 0x2390 == 0xffffeef8 +# CHECK: Hex dump of section '.data': +# CHECK-NEXT: eef8ef20 ffffeef8 ffffef20 ffffffff +# CHECK-NEXT: ffffeef8 ffffffff ffffef20 + +bar: + br %r14 + +.data +.reloc ., R_390_GOTOFF16, bar +.space 2 +.reloc ., R_390_PLTOFF16, foo +.space 2 +.reloc ., R_390_GOTOFF, bar +.space 4 +.reloc ., R_390_PLTOFF32, foo +.space 4 +.reloc ., R_390_GOTOFF64, bar +.space 8 +.reloc ., R_390_PLTOFF64, foo +.space 8 diff --git a/lld/test/ELF/systemz-reloc-pc16.s b/lld/test/ELF/systemz-reloc-pc16.s new file mode 100644 index 000000000000..e1dad5af239d --- /dev/null +++ b/lld/test/ELF/systemz-reloc-pc16.s @@ -0,0 +1,39 @@ +# REQUIRES: systemz +# RUN: rm -rf %t && split-file %s %t + +## Check recompile with -fPIC error message +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %t/shared.s -o %t/shared.o +# RUN: not ld.lld -shared %t/shared.o -o /dev/null 2>&1 | FileCheck %s + +# CHECK: error: relocation R_390_PC16 cannot be used against symbol '_shared'; recompile with -fPIC +# CHECK: >>> defined in {{.*}} +# CHECK: >>> referenced by {{.*}}:(.data+0x1) + +## Check patching of negative addends + +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym ADDEND=1 %t/addend.s -o %t/1.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym ADDEND=32768 %t/addend.s -o %t/2.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym ADDEND=32769 %t/addend.s -o %t/3.o + +# RUN: ld.lld --section-start=.text=0x0 %t/1.o -o %t/1out +# RUN: ld.lld --section-start=.text=0x0 %t/2.o -o %t/2out +# RUN: not ld.lld --section-start=.text=0x0 %t/3.o -o /dev/null 2>&1 | FileCheck %s -DFILE=%t/3.o --check-prefix RANGE + +# RANGE: error: [[FILE]]:(.text+0x0): relocation R_390_PC16 out of range + +# RUN: llvm-readelf --hex-dump=.text %t/1out | FileCheck %s -DADDEND=ffff --check-prefix DUMP +# RUN: llvm-readelf --hex-dump=.text %t/2out | FileCheck %s -DADDEND=8000 --check-prefix DUMP + +# DUMP: 0x00000000 [[ADDEND]] + +#--- shared.s +.data + .byte 0xe8 + .word _shared - . + +#--- addend.s +.text +.globl _start +_start: + .reloc ., R_390_PC16, .text-ADDEND + .space 2 diff --git a/lld/test/ELF/systemz-reloc-pc32.s b/lld/test/ELF/systemz-reloc-pc32.s new file mode 100644 index 000000000000..0cb9322eb1c1 --- /dev/null +++ b/lld/test/ELF/systemz-reloc-pc32.s @@ -0,0 +1,39 @@ +# REQUIRES: systemz +# RUN: rm -rf %t && split-file %s %t + +## Check recompile with -fPIC error message +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %t/shared.s -o %t/shared.o +# RUN: not ld.lld -shared %t/shared.o -o /dev/null 2>&1 | FileCheck %s + +# CHECK: error: relocation R_390_PC32 cannot be used against symbol '_shared'; recompile with -fPIC +# CHECK: >>> defined in {{.*}} +# CHECK: >>> referenced by {{.*}}:(.data+0x1) + +## Check patching of negative addends + +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym ADDEND=1 %t/addend.s -o %t/1.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym ADDEND=2147483648 %t/addend.s -o %t/2.o +# RUN: llvm-mc -filetype=obj -triple=s390x -defsym ADDEND=2147483649 %t/addend.s -o %t/3.o + +# RUN: ld.lld --section-start=.text=0x0 %t/1.o -o %t/1out +# RUN: ld.lld --section-start=.text=0x0 %t/2.o -o %t/2out +# RUN: not ld.lld --section-start=.text=0x0 %t/3.o -o /dev/null 2>&1 | FileCheck %s -DFILE=%t/3.o --check-prefix RANGE + +# RANGE: error: [[FILE]]:(.text+0x0): relocation R_390_PC32 out of range + +# RUN: llvm-readelf --hex-dump=.text %t/1out | FileCheck %s -DADDEND=ffffffff --check-prefix DUMP +# RUN: llvm-readelf --hex-dump=.text %t/2out | FileCheck %s -DADDEND=80000000 --check-prefix DUMP + +# DUMP: 0x00000000 [[ADDEND]] + +#--- shared.s +.data + .byte 0xe8 + .long _shared - . + +#--- addend.s +.text +.globl _start +_start: + .reloc ., R_390_PC32, .text-ADDEND + .space 4 diff --git a/lld/test/ELF/systemz-reloc-pcdbl.s b/lld/test/ELF/systemz-reloc-pcdbl.s new file mode 100644 index 000000000000..faee756f5e95 --- /dev/null +++ b/lld/test/ELF/systemz-reloc-pcdbl.s @@ -0,0 +1,68 @@ +# REQUIRES: systemz + +# RUN: llvm-mc --filetype=obj --triple=s390x-unknown-linux -mcpu=z13 %s -o %t.o + +# RUN: ld.lld %t.o --defsym foo16=pc16dbl+4 --defsym bar16=pc16dbl --defsym foo32=pc32dbl+6 --defsym bar32=pc32dbl --defsym foo12=pc12dbl+6 --defsym bar12=pc12dbl --defsym foo24=pc24dbl+6 --defsym bar24=pc24dbl -o %t +# RUN: llvm-objdump --no-show-raw-insn --mcpu=z13 -d %t | FileCheck %s --check-prefix=CHECK +# CHECK: 0000000001001120 : +# CHECK: je 0x1001124 +# CHECK: jne 0x1001120 +# CHECK: 0000000001001128 : +# CHECK: jge 0x100112e +# CHECK: jgne 0x1001128 +# CHECK: 0000000001001134 : +# CHECK: bprp 5, 0x100113a, 0x1001134 +# CHECK: bprp 6, 0x1001134, 0x100113a +# CHECK: 0000000001001140 : +# CHECK: bprp 5, 0x1001140, 0x1001146 +# CHECK: bprp 6, 0x1001146, 0x1001140 + +# RUN: ld.lld %t.o --defsym foo16=pc16dbl+0xfffe --defsym bar16=pc16dbl+4-0x10000 --defsym foo32=pc32dbl+0xfffffffe --defsym bar32=pc32dbl+6-0x100000000 --defsym foo12=pc12dbl+0xffe --defsym bar12=pc12dbl+6-0x1000 --defsym foo24=pc24dbl+0xfffffe --defsym bar24=pc24dbl+6-0x1000000 -o %t.limits +# RUN: llvm-objdump --no-show-raw-insn --mcpu=z13 -d %t.limits | FileCheck %s --check-prefix=LIMITS +# LIMITS: je 0x101111e +# LIMITS-NEXT: jne 0xff1124 +# LIMITS: jge 0x101001126 +# LIMITS-NEXT: jgne 0xffffffff0100112e +# LIMITS: bprp 5, 0x1002132, 0x1001134 +# LIMITS-NEXT: bprp 6, 0x100013a, 0x100113a +# LIMITS: bprp 5, 0x1001140, 0x200113e +# LIMITS-NEXT: bprp 6, 0x1001146, 0x1146 + +# RUN: not ld.lld %t.o --defsym foo16=pc16dbl+0x10000 --defsym bar16=pc16dbl+4-0x10002 --defsym foo32=pc32dbl+0x100000000 --defsym bar32=pc32dbl+6-0x100000002 --defsym foo12=pc12dbl+0x1000 --defsym bar12=pc12dbl+6-0x1002 --defsym foo24=pc24dbl+0x1000000 --defsym bar24=pc24dbl+6-0x1000002 -o /dev/null 2>&1 | FileCheck -DFILE=%t.o --check-prefix=ERROR-RANGE %s +# ERROR-RANGE: error: [[FILE]]:(.text+0x2): relocation R_390_PC16DBL out of range: 65536 is not in [-65536, 65535]; references 'foo16' +# ERROR-RANGE: error: [[FILE]]:(.text+0x6): relocation R_390_PC16DBL out of range: -65538 is not in [-65536, 65535]; references 'bar16' +# ERROR-RANGE: error: [[FILE]]:(.text+0xa): relocation R_390_PC32DBL out of range: 4294967296 is not in [-4294967296, 4294967295]; references 'foo32' +# ERROR-RANGE: error: [[FILE]]:(.text+0x10): relocation R_390_PC32DBL out of range: -4294967298 is not in [-4294967296, 4294967295]; references 'bar32' +# ERROR-RANGE: error: [[FILE]]:(.text+0x15): relocation R_390_PC12DBL out of range: 4096 is not in [-4096, 4095]; references 'foo12' +# ERROR-RANGE: error: [[FILE]]:(.text+0x1b): relocation R_390_PC12DBL out of range: -4098 is not in [-4096, 4095]; references 'bar12' +# ERROR-RANGE: error: [[FILE]]:(.text+0x23): relocation R_390_PC24DBL out of range: 16777216 is not in [-16777216, 16777215]; references 'foo24' +# ERROR-RANGE: error: [[FILE]]:(.text+0x29): relocation R_390_PC24DBL out of range: -16777218 is not in [-16777216, 16777215]; references 'bar24' + +# RUN: not ld.lld %t.o --defsym foo16=pc16dbl+1 --defsym bar16=pc16dbl-1 --defsym foo32=pc32dbl+1 --defsym bar32=pc32dbl-1 --defsym foo12=pc12dbl+1 --defsym bar12=pc12dbl-1 --defsym foo24=pc24dbl+1 --defsym bar24=pc24dbl-1 -o /dev/null 2>&1 | FileCheck -DFILE=%t.o --check-prefix=ERROR-ALIGN %s +# ERROR-ALIGN: error: [[FILE]]:(.text+0x2): improper alignment for relocation R_390_PC16DBL: 0x1 is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0x6): improper alignment for relocation R_390_PC16DBL: 0xFFFFFFFFFFFFFFFB is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0xa): improper alignment for relocation R_390_PC32DBL: 0x1 is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0x10): improper alignment for relocation R_390_PC32DBL: 0xFFFFFFFFFFFFFFF9 is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0x15): improper alignment for relocation R_390_PC12DBL: 0x1 is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0x1b): improper alignment for relocation R_390_PC12DBL: 0xFFFFFFFFFFFFFFF9 is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0x23): improper alignment for relocation R_390_PC24DBL: 0x1 is not aligned to 2 bytes +# ERROR-ALIGN-NEXT: error: [[FILE]]:(.text+0x29): improper alignment for relocation R_390_PC24DBL: 0xFFFFFFFFFFFFFFF9 is not aligned to 2 bytes + +.global _start +.global pc16dbl +.global pc32dbl +.global pc12dbl +.global pc24dbl +_start: +pc16dbl: + je foo16 + jne bar16 +pc32dbl: + jge foo32 + jgne bar32 +pc12dbl: + bprp 5,foo12,0 + bprp 6,bar12,0 +pc24dbl: + bprp 5,0,foo24 + bprp 6,0,bar24 diff --git a/lld/test/ELF/systemz-tls-gd.s b/lld/test/ELF/systemz-tls-gd.s new file mode 100644 index 000000000000..3976f55a6ae3 --- /dev/null +++ b/lld/test/ELF/systemz-tls-gd.s @@ -0,0 +1,142 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: echo '.tbss; .globl b, c; b: .zero 4; c:' | llvm-mc -filetype=obj -triple=s390x-unknown-linux - -o %t1.o +# RUN: ld.lld -shared -soname=t1.so %t1.o -o %t1.so + +# RUN: ld.lld -shared %t.o %t1.o -o %t.so +# RUN: llvm-readelf -r %t.so | FileCheck --check-prefix=GD-REL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck --check-prefix=GD %s +# RUN: llvm-objdump --section .data.rel.ro --full-contents %t.so | FileCheck --check-prefix=GD-DATA %s + +# RUN: ld.lld %t.o %t1.o -o %t.le +# RUN: llvm-readelf -r %t.le | FileCheck --check-prefix=NOREL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t.le | FileCheck --check-prefix=LE %s +# RUN: llvm-objdump --section .data.rel.ro --full-contents %t.le | FileCheck --check-prefix=LE-DATA %s + +# RUN: ld.lld %t.o %t1.so -o %t.ie +# RUN: llvm-readelf -r %t.ie | FileCheck --check-prefix=IE-REL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t.ie | FileCheck --check-prefix=IE %s +# RUN: llvm-objdump --section .data.rel.ro --full-contents %t.ie | FileCheck --check-prefix=IE-DATA %s + +# GD-REL: Relocation section '.rela.dyn' at offset {{.*}} contains 6 entries: +# GD-REL: 0000000000002570 0000000200000036 R_390_TLS_DTPMOD 0000000000000008 a + 0 +# GD-REL-NEXT: 0000000000002578 0000000200000037 R_390_TLS_DTPOFF 0000000000000008 a + 0 +# GD-REL-NEXT: 0000000000002580 0000000300000036 R_390_TLS_DTPMOD 000000000000000c b + 0 +# GD-REL-NEXT: 0000000000002588 0000000300000037 R_390_TLS_DTPOFF 000000000000000c b + 0 +# GD-REL-NEXT: 0000000000002590 0000000400000036 R_390_TLS_DTPMOD 0000000000000010 c + 0 +# GD-REL-NEXT: 0000000000002598 0000000400000037 R_390_TLS_DTPOFF 0000000000000010 c + 0 + +## _GLOBAL_OFFSET_TABLE is at 0x2558 +# GD: larl %r12, 0x2558 + +## GOT offset of the TLS module ID / offset pair for a is at 0x2460 +# GD-NEXT: lgrl %r2, 0x2460 +# GD-NEXT: brasl %r14, 0x1440 +# GD-NEXT: lgf %r2, 0(%r2,%r7) + +## GOT offset of the TLS module ID / offset pair for b is at 0x2468 +# GD-NEXT: lgrl %r2, 0x2468 +# GD-NEXT: brasl %r14, 0x1440 +# GD-NEXT: lgf %r2, 0(%r2,%r7) + +## GOT offset of the TLS module ID / offset pair for c is at 0x2470 +# GD-NEXT: lgrl %r2, 0x2470 +# GD-NEXT: brasl %r14, 0x1440 +# GD-NEXT: lgf %r2, 0(%r2,%r7) + +## Constant pool holding GOT offsets of TLS module ID / offset pairs: +# a: 0x2570 / 0x18 +# b: 0x2580 / 0x28 +# c: 0x2590 / 0x38 +# GD-DATA: 2460 00000000 00000018 00000000 00000028 +# GD-DATA-NEXT: 2470 00000000 00000038 + +# NOREL: no relocations + +## _GLOBAL_OFFSET_TABLE is at 0x1002230 +# LE: larl %r12, 0x1002230 + +## TP offset of a is at 0x1002218 +# LE-NEXT: lgrl %r2, 0x1002218 +# LE-NEXT: brcl 0, +# LE-NEXT: lgf %r2, 0(%r2,%r7) + +## TP offset of b is at 0x1002220 +# LE-NEXT: lgrl %r2, 0x1002220 +# LE-NEXT: brcl 0, +# LE-NEXT: lgf %r2, 0(%r2,%r7) + +## TP offset of c is at 0x1002228 +# LE-NEXT: lgrl %r2, 0x1002228 +# LE-NEXT: brcl 0, +# LE-NEXT: lgf %r2, 0(%r2,%r7) + +## TP offsets +# a: -8 +# b: -4 +# c: 0 +# LE-DATA: 1002218 ffffffff fffffff8 ffffffff fffffffc +# LE-DATA-NEXT: 1002228 00000000 00000000 + + +# IE-REL: Relocation section '.rela.dyn' at offset {{.*}} contains 2 entries: +# IE-REL: 0000000001002430 0000000200000038 R_390_TLS_TPOFF 0000000000000000 b + 0 +# IE-REL-NEXT: 0000000001002438 0000000300000038 R_390_TLS_TPOFF 0000000000000000 c + 0 + +## _GLOBAL_OFFSET_TABLE is at 0x1002418 +# IE: larl %r12, 0x1002418 + +## TP offset of a is at 0x1002340 +# IE-NEXT: lgrl %r2, 0x1002340 +# IE-NEXT: brcl 0, +# IE-NEXT: lgf %r2, 0(%r2,%r7) + +## GOT offset of the TP offset for b is at 0x1002348 +# IE-NEXT: lgrl %r2, 0x1002348 +# IE-NEXT: lg %r2, 0(%r2,%r12) +# IE-NEXT: lgf %r2, 0(%r2,%r7) + +## GOT offset of the TP offset for c is at 0x1002350 +# IE-NEXT: lgrl %r2, 0x1002350 +# IE-NEXT: lg %r2, 0(%r2,%r12) +# IE-NEXT: lgf %r2, 0(%r2,%r7) + +## TP offsets (a) / GOT offset of TP offsets (b, c) +# a: -4 +# b: 0x1002430 / 0x18 +# c: 0x1002438 / 0x20 +# IE-DATA: 1002340 ffffffff fffffffc 00000000 00000018 +# IE-DATA-NEXT: 1002350 00000000 00000020 + + +ear %r7,%a0 +sllg %r7,%r1,32 +ear %r7,%a1 +larl %r12,_GLOBAL_OFFSET_TABLE_ + +lgrl %r2,.LC0 +brasl %r14,__tls_get_offset@PLT:tls_gdcall:a +lgf %r2,0(%r2,%r7) + +lgrl %r2,.LC1 +brasl %r14,__tls_get_offset@PLT:tls_gdcall:b +lgf %r2,0(%r2,%r7) + +lgrl %r2,.LC2 +brasl %r14,__tls_get_offset@PLT:tls_gdcall:c +lgf %r2,0(%r2,%r7) + + .section .data.rel.ro,"aw" + .align 8 +.LC0: + .quad a@TLSGD +.LC1: + .quad b@TLSGD +.LC2: + .quad c@TLSGD + + .section .tbss + .globl a + .zero 8 +a: + .zero 4 diff --git a/lld/test/ELF/systemz-tls-ie.s b/lld/test/ELF/systemz-tls-ie.s new file mode 100644 index 000000000000..27b642ed2dfc --- /dev/null +++ b/lld/test/ELF/systemz-tls-ie.s @@ -0,0 +1,87 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o + +# RUN: ld.lld -shared %t.o -o %t.so +# RUN: llvm-readelf -r %t.so | FileCheck --check-prefix=IE-REL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck --check-prefix=IE %s +# RUN: llvm-objdump --section .data --full-contents %t.so | FileCheck --check-prefix=IE-DATA %s + +# RUN: ld.lld %t.o -o %t +# RUN: llvm-readelf -r %t | FileCheck --check-prefix=NOREL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=LE %s +# RUN: llvm-objdump --section .data --full-contents %t | FileCheck --check-prefix=LE-DATA %s +# RUN: llvm-objdump --section .got --full-contents %t | FileCheck --check-prefix=LE-GOT %s + +# IE-REL: Relocation section '.rela.dyn' at offset {{.*}} contains 4 entries: +# IE-REL: 0000000000003478 000000000000000c R_390_RELATIVE 2460 +# IE-REL: 0000000000002460 0000000100000038 R_390_TLS_TPOFF 0000000000000008 a + 0 +# IE-REL: 0000000000002468 0000000200000038 R_390_TLS_TPOFF 000000000000000c b + 0 +# IE-REL: 0000000000002470 0000000300000038 R_390_TLS_TPOFF 0000000000000010 c + 0 + +## TP offset for a is at 0x2460 +# IE: lgrl %r1, 0x2460 +# IE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offset for b is at 0x2468 +# IE-NEXT: lgrl %r1, 0x2468 +# IE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offset for c is at 0x2470 +# IE-NEXT: lgrl %r1, 0x2470 +# IE-NEXT: lgf %r1, 0(%r1,%r7) + +## Data element: TP offset for a is at 0x2460 (relocated via R_390_RELATIVE above) +# IE-DATA: 3478 00000000 00000000 + +# NOREL: no relocations + +## TP offset for a is at 0x1002250 +# LE: lgrl %r1, 0x1002250 +# LE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offset for b is at 0x1002258 +# LE-NEXT: lgrl %r1, 0x1002258 +# LE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offset for c is at 0x1002260 +# LE-NEXT: lgrl %r1, 0x1002260 +# LE-NEXT: lgf %r1, 0(%r1,%r7) + +## Data element: TP offset for a is at 0x1002250 +# LE-DATA: 00000000 01002250 + +## TP offsets in GOT: +# a: -8 +# b: -4 +# c: 0 +# LE-GOT: 1002238 00000000 00000000 00000000 00000000 +# LE-GOT: 1002248 00000000 00000000 ffffffff fffffff8 +# LE-GOT: 1002258 ffffffff fffffffc 00000000 00000000 + +ear %r7,%a0 +sllg %r7,%r1,32 +ear %r7,%a1 + +lgrl %r1, a@indntpoff +lgf %r1,0(%r1,%r7) + +lgrl %r1, b@indntpoff +lgf %r1,0(%r1,%r7) + +lgrl %r1, c@indntpoff +lgf %r1,0(%r1,%r7) + + .data + .reloc .,R_390_TLS_IE64,a + .space 8 + + .section .tbss + .globl a + .globl b + .globl c + .zero 8 +a: + .zero 4 +b: + .zero 4 +c: diff --git a/lld/test/ELF/systemz-tls-ld.s b/lld/test/ELF/systemz-tls-ld.s new file mode 100644 index 000000000000..2cb36d7294f2 --- /dev/null +++ b/lld/test/ELF/systemz-tls-ld.s @@ -0,0 +1,114 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o + +# RUN: ld.lld -shared %t.o -o %t.so +# RUN: llvm-readelf -r %t.so | FileCheck --check-prefix=LD-REL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck --check-prefix=LD %s +# RUN: llvm-objdump --section .data.rel.ro --full-contents %t.so | FileCheck --check-prefix=LD-DATA %s + +# RUN: ld.lld %t.o -o %t +# RUN: llvm-readelf -r %t | FileCheck --check-prefix=NOREL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=LE %s +# RUN: llvm-objdump --section .data.rel.ro --full-contents %t | FileCheck --check-prefix=LE-DATA %s + +# LD-REL: Relocation section '.rela.dyn' at offset {{.*}} contains 1 entries: +# LD-REL: 00000000000024f8 0000000000000036 R_390_TLS_DTPMOD 0 + +## _GLOBAL_OFFSET_TABLE is at 0x24e0 +# LD: larl %r12, 0x24e0 + +## GOT offset of the LDM TLS module ID is at 0x23e0 +# LD-NEXT: lgrl %r2, 0x23e0 +# LD-NEXT: brasl %r14, 0x13c0 +# LD-NEXT: la %r2, 0(%r2,%r7) + +## DTP offset for a is at 0x23e8 +# LD-NEXT: lgrl %r1, 0x23e8 +# LD-NEXT: lgf %r1, 0(%r1,%r2) + +## DTP offset for b is at 0x23f0 +# LD-NEXT: lgrl %r1, 0x23f0 +# LD-NEXT: lgf %r1, 0(%r1,%r2) + +## DTP offset for c is at 0x23f8 +# LD-NEXT: lgrl %r1, 0x23f8 +# LD-NEXT: lgf %r1, 0(%r1,%r2) + +## Constant pool holding GOT offsets of TLS module ID and DTP offsets: +# TLS module ID: 0x24f8 / 0x18 +# a: 8 +# b: 12 +# c: 16 +# LD-DATA: 23e0 00000000 00000018 00000000 00000008 +# LD-DATA: 23f0 00000000 0000000c 00000000 00000010 + +# NOREL: no relocations + +## _GLOBAL_OFFSET_TABLE is at 0x1002230 +# LE: larl %r12, 0x1002230 + +## GOT offset of the LDM TLS module ID is at 0x1002210 +# LE-NEXT: lgrl %r2, 0x1002210 +# LE-NEXT: brcl 0, +# LE-NEXT: la %r2, 0(%r2,%r7) + +## TP offset for a is at 0x1002218 +# LE-NEXT: lgrl %r1, 0x1002218 +# LE-NEXT: lgf %r1, 0(%r1,%r2) + +## TP offset for b is at 0x1002220 +# LE-NEXT: lgrl %r1, 0x1002220 +# LE-NEXT: lgf %r1, 0(%r1,%r2) + +## TP offset for c is at 0x1002228 +# LE-NEXT: lgrl %r1, 0x1002228 +# LE-NEXT: lgf %r1, 0(%r1,%r2) + +## zeroed LDM / TP offsets: +# LDM TLS: 0 +# a: -8 +# b: -4 +# c: 0 +# LE-DATA: 1002210 00000000 00000000 ffffffff fffffff8 +# LE-DATA: 1002220 ffffffff fffffffc 00000000 00000000 + + +ear %r7,%a0 +sllg %r7,%r1,32 +ear %r7,%a1 +larl %r12,_GLOBAL_OFFSET_TABLE_ + +lgrl %r2,.LC0 +brasl %r14,__tls_get_offset@PLT:tls_ldcall:a +la %r2,0(%r2,%r7) + +lgrl %r1, .LC1 +lgf %r1,0(%r1,%r2) + +lgrl %r1, .LC2 +lgf %r1,0(%r1,%r2) + +lgrl %r1, .LC3 +lgf %r1,0(%r1,%r2) + + .section .data.rel.ro,"aw" + .align 8 +.LC0: + .quad a@TLSLDM +.LC1: + .quad a@DTPOFF +.LC2: + .quad b@DTPOFF +.LC3: + .quad c@DTPOFF + + .section .tbss + .globl a + .globl b + .globl c + .zero 8 +a: + .zero 4 +b: + .zero 4 +c: diff --git a/lld/test/ELF/systemz-tls-le.s b/lld/test/ELF/systemz-tls-le.s new file mode 100644 index 000000000000..9e41fc768da3 --- /dev/null +++ b/lld/test/ELF/systemz-tls-le.s @@ -0,0 +1,61 @@ +# REQUIRES: systemz +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o + +# RUN: ld.lld %t.o -o %t +# RUN: llvm-readelf -r %t | FileCheck --check-prefix=NOREL %s +# RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=LE %s +# RUN: llvm-objdump --section .data.rel.ro --full-contents %t | FileCheck --check-prefix=LE-DATA %s + +# NOREL: no relocations + +## TP offset for a is at 0x1002200 +# LE: lgrl %r1, 0x1002200 +# LE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offset for b is at 0x1002208 +# LE-NEXT: lgrl %r1, 0x1002208 +# LE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offset for c is at 0x1002210 +# LE-NEXT: lgrl %r1, 0x1002210 +# LE-NEXT: lgf %r1, 0(%r1,%r7) + +## TP offsets: +# a: -8 +# b: -4 +# c: 0 +# LE-DATA: 1002200 ffffffff fffffff8 ffffffff fffffffc +# LE-DATA: 1002210 00000000 00000000 + +ear %r7,%a0 +sllg %r7,%r1,32 +ear %r7,%a1 + +lgrl %r1, .LC0 +lgf %r1,0(%r1,%r7) + +lgrl %r1, .LC1 +lgf %r1,0(%r1,%r7) + +lgrl %r1, .LC2 +lgf %r1,0(%r1,%r7) + + .section .data.rel.ro,"aw" + .align 8 +.LC0: + .quad a@ntpoff +.LC1: + .quad b@ntpoff +.LC2: + .quad c@ntpoff + + .section .tbss + .globl a + .globl b + .globl c + .zero 8 +a: + .zero 4 +b: + .zero 4 +c: diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py index b3e07f1f823c..d309c2ad4ee2 100644 --- a/lld/test/lit.cfg.py +++ b/lld/test/lit.cfg.py @@ -83,6 +83,7 @@ llvm_config.feature_config( "PowerPC": "ppc", "RISCV": "riscv", "Sparc": "sparc", + "SystemZ": "systemz", "WebAssembly": "wasm", "X86": "x86", }, -- GitLab From 9ca1a1575a337931d0e49859f83a0d5b70916abd Mon Sep 17 00:00:00 2001 From: David Spickett Date: Tue, 13 Feb 2024 10:38:38 +0000 Subject: [PATCH 043/284] [flang][Driver] Add -masm option to flang (#81490) The motivation here was a suggestion over in Compiler Explorer. You can use `-mllvm` already to do this but since gfortran supports `-masm`, I figured I'd try to add it. This is done by flang expanding `-masm` into `-mllvm x86-asm-syntax=`, then passing that to fc1. Which then collects all the `-mllvm` options and forwards them on. The code to expand it comes from clang `Clang::AddX86TargetArgs` (there are some other places doing the same thing too). However I've removed the `-inline-asm` that clang adds, as fortran doesn't have inline assembly. So `-masm` for flang purely changes the style of assembly output. ``` $ ./bin/flang-new /tmp/test.f90 -o - -S -target x86_64-linux-gnu <...> pushq %rbp $ ./bin/flang-new /tmp/test.f90 -o - -S -target x86_64-linux-gnu -masm=att <...> pushq %rbp $ ./bin/flang-new /tmp/test.f90 -o - -S -target x86_64-linux-gnu -masm=intel <...> push rbp ``` The test is adapted from `clang/test/Driver/masm.c` by removing the clang-cl related lines and changing the 32 bit triples to 64 bit triples since flang doesn't support 32 bit targets. --- clang/include/clang/Driver/Options.td | 2 +- clang/lib/Driver/ToolChains/Flang.cpp | 15 +++++++++++++++ clang/lib/Driver/ToolChains/Flang.h | 7 +++++++ flang/test/Driver/masm.f90 | 10 ++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 flang/test/Driver/masm.f90 diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 31e8571758bf..d5017b901d29 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -4414,7 +4414,7 @@ def mwatchsimulator_version_min_EQ : Joined<["-"], "mwatchsimulator-version-min= def march_EQ : Joined<["-"], "march=">, Group, Flags<[TargetSpecific]>, Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>, HelpText<"For a list of available architectures for the target use '-mcpu=help'">; -def masm_EQ : Joined<["-"], "masm=">, Group; +def masm_EQ : Joined<["-"], "masm=">, Group, Visibility<[ClangOption, FlangOption]>; def inline_asm_EQ : Joined<["-"], "inline-asm=">, Group, Visibility<[ClangOption, CC1Option]>, Values<"att,intel">, diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp index 23da08aa593f..6168b42dc782 100644 --- a/clang/lib/Driver/ToolChains/Flang.cpp +++ b/clang/lib/Driver/ToolChains/Flang.cpp @@ -249,6 +249,20 @@ void Flang::AddRISCVTargetArgs(const ArgList &Args, } } +void Flang::AddX86_64TargetArgs(const ArgList &Args, + ArgStringList &CmdArgs) const { + if (Arg *A = Args.getLastArg(options::OPT_masm_EQ)) { + StringRef Value = A->getValue(); + if (Value == "intel" || Value == "att") { + CmdArgs.push_back(Args.MakeArgString("-mllvm")); + CmdArgs.push_back(Args.MakeArgString("-x86-asm-syntax=" + Value)); + } else { + getToolChain().getDriver().Diag(diag::err_drv_unsupported_option_argument) + << A->getSpelling() << Value; + } + } +} + static void addVSDefines(const ToolChain &TC, const ArgList &Args, ArgStringList &CmdArgs) { @@ -374,6 +388,7 @@ void Flang::addTargetOptions(const ArgList &Args, break; case llvm::Triple::x86_64: getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false); + AddX86_64TargetArgs(Args, CmdArgs); break; } diff --git a/clang/lib/Driver/ToolChains/Flang.h b/clang/lib/Driver/ToolChains/Flang.h index ec2e545a1d0b..9f5e26b86083 100644 --- a/clang/lib/Driver/ToolChains/Flang.h +++ b/clang/lib/Driver/ToolChains/Flang.h @@ -77,6 +77,13 @@ private: void AddRISCVTargetArgs(const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs) const; + /// Add specific options for X86_64 target. + /// + /// \param [in] Args The list of input driver arguments + /// \param [out] CmdArgs The list of output command arguments + void AddX86_64TargetArgs(const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CmdArgs) const; + /// Extract offload options from the driver arguments and add them to /// the command arguments. /// \param [in] C The current compilation for the driver invocation diff --git a/flang/test/Driver/masm.f90 b/flang/test/Driver/masm.f90 new file mode 100644 index 000000000000..c5c44efeb3c8 --- /dev/null +++ b/flang/test/Driver/masm.f90 @@ -0,0 +1,10 @@ +! RUN: %flang --target=x86_64-unknown-linux -masm=intel -S %s -### 2>&1 | FileCheck --check-prefix=CHECK-INTEL %s +! RUN: %flang --target=x86_64-unknown-linux -masm=att -S %s -### 2>&1 | FileCheck --check-prefix=CHECK-ATT %s +! RUN: not %flang --target=x86_64-unknown-linux -S -masm=somerequired %s -### 2>&1 | FileCheck --check-prefix=CHECK-SOMEREQUIRED %s +! RUN: %flang --target=aarch64-unknown-eabi -S -masm=intel %s -### 2>&1 | FileCheck --check-prefix=CHECK-AARCH64 %s + +! CHECK-INTEL: "-mllvm" "-x86-asm-syntax=intel" +! CHECK-ATT: "-mllvm" "-x86-asm-syntax=att" +! CHECK-SOMEREQUIRED: error: unsupported argument 'somerequired' to option '-masm=' +! CHECK-AARCH64: warning: argument unused during compilation: '-masm=intel' +! CHECK-AARCH64-NOT: -x86-asm-syntax=intel -- GitLab From a8fb0dcc41bf355a3bff9ad7165715a8b6012059 Mon Sep 17 00:00:00 2001 From: Paul Semel Date: Tue, 13 Feb 2024 11:39:27 +0100 Subject: [PATCH 044/284] [dataflow] CXXOperatorCallExpr equal operator might not be a glvalue (#80991) Although in a normal implementation the assumption is reasonable, it seems that some esoteric implementation are not returning a T&. This should be handled correctly and the values be propagated. --------- Co-authored-by: martinboehme --- clang/lib/Analysis/FlowSensitive/Transfer.cpp | 14 +++++++- .../Analysis/FlowSensitive/TransferTest.cpp | 36 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp index a098471d0ee9..f0b15f43b1f4 100644 --- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp +++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp @@ -535,7 +535,19 @@ public: return; copyRecord(*LocSrc, *LocDst, Env); - Env.setStorageLocation(*S, *LocDst); + + // If the expr is a glvalue, we can reasonably assume the operator is + // returning T& and thus we can assign it `LocDst`. + if (S->isGLValue()) { + Env.setStorageLocation(*S, *LocDst); + } else if (S->getType()->isRecordType()) { + // Make sure that we have a `RecordValue` for this expression so that + // `Environment::getResultObjectLocation()` is able to return a location + // for it. + if (Env.getValue(*S) == nullptr) + refreshRecordValue(*S, Env); + } + return; } diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index 55af70289ee0..4b3b3511f848 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -2313,6 +2313,42 @@ TEST(TransferTest, AssignmentOperatorWithInitAndInheritance) { ASTContext &ASTCtx) {}); } +TEST(TransferTest, AssignmentOperatorReturnsVoid) { + // This is a crash repro. + std::string Code = R"( + struct S { + void operator=(S&& other); + }; + void target() { + S s; + s = S(); + // [[p]] + } + )"; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) {}); +} + +TEST(TransferTest, AssignmentOperatorReturnsByValue) { + // This is a crash repro. + std::string Code = R"( + struct S { + S operator=(S&& other); + }; + void target() { + S s; + s = S(); + // [[p]] + } + )"; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) {}); +} + TEST(TransferTest, CopyConstructor) { std::string Code = R"( struct A { -- GitLab From 79ce2c93aeb4686ef687b19867dbfe0e8cf40673 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Tue, 13 Feb 2024 10:47:33 +0000 Subject: [PATCH 045/284] [mlir][VectorOps] Add conversion of 1-D vector.interleave ops to LLVM (#80966) The 1-D case directly maps to LLVM intrinsics. The n-D case will be handled by unrolling to 1-D first (in a later patch). Depends on: #80965 --- .../VectorToLLVM/ConvertVectorToLLVM.cpp | 42 ++++++++++++++++++- .../VectorToLLVM/vector-to-llvm.mlir | 37 ++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp index b66b55ae8d57..19cc914efae0 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp @@ -1734,6 +1734,45 @@ struct VectorSplatNdOpLowering : public ConvertOpToLLVMPattern { } }; +/// Conversion pattern for a `vector.interleave`. +/// This supports fixed-sized vectors and scalable vectors. +struct VectorInterleaveOpLowering + : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(vector::InterleaveOp interleaveOp, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + VectorType resultType = interleaveOp.getResultVectorType(); + // n-D interleaves should have been lowered already. + if (resultType.getRank() != 1) + return rewriter.notifyMatchFailure(interleaveOp, + "InterleaveOp not rank 1"); + // If the result is rank 1, then this directly maps to LLVM. + if (resultType.isScalable()) { + rewriter.replaceOpWithNewOp( + interleaveOp, typeConverter->convertType(resultType), + adaptor.getLhs(), adaptor.getRhs()); + return success(); + } + // Lower fixed-size interleaves to a shufflevector. While the + // vector.interleave2 intrinsic supports fixed and scalable vectors, the + // langref still recommends fixed-vectors use shufflevector, see: + // https://llvm.org/docs/LangRef.html#id876. + int64_t resultVectorSize = resultType.getNumElements(); + SmallVector interleaveShuffleMask; + interleaveShuffleMask.reserve(resultVectorSize); + for (int i = 0, end = resultVectorSize / 2; i < end; ++i) { + interleaveShuffleMask.push_back(i); + interleaveShuffleMask.push_back((resultVectorSize / 2) + i); + } + rewriter.replaceOpWithNewOp( + interleaveOp, adaptor.getLhs(), adaptor.getRhs(), + interleaveShuffleMask); + return success(); + } +}; + } // namespace /// Populate the given list with patterns that convert from Vector to LLVM. @@ -1758,7 +1797,8 @@ void mlir::populateVectorToLLVMConversionPatterns( VectorExpandLoadOpConversion, VectorCompressStoreOpConversion, VectorSplatOpLowering, VectorSplatNdOpLowering, VectorScalableInsertOpLowering, VectorScalableExtractOpLowering, - MaskedReductionOpConversion>(converter); + MaskedReductionOpConversion, VectorInterleaveOpLowering>( + converter); // Transfer ops with rank > 1 are handled by VectorToSCF. populateVectorTransferLoweringPatterns(patterns, /*maxTransferRank=*/1); } diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir index 1c13b16dfd9a..a46f2e101f3c 100644 --- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir @@ -2460,3 +2460,40 @@ func.func @make_fixed_vector_of_scalable_vector(%f : f64) -> vector<3x[2]xf64> %res = vector.broadcast %f : f64 to vector<3x[2]xf64> return %res : vector<3x[2]xf64> } + +// ----- + +// CHECK-LABEL: @vector_interleave_0d +// CHECK-SAME: %[[LHS:.*]]: vector, %[[RHS:.*]]: vector) +func.func @vector_interleave_0d(%a: vector, %b: vector) -> vector<2xi8> { + // CHECK: %[[LHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[LHS]] : vector to vector<1xi8> + // CHECK: %[[RHS_RANK1:.*]] = builtin.unrealized_conversion_cast %[[RHS]] : vector to vector<1xi8> + // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS_RANK1]], %[[RHS_RANK1]] [0, 1] : vector<1xi8> + // CHECK: return %[[ZIP]] + %0 = vector.interleave %a, %b : vector + return %0 : vector<2xi8> +} + +// ----- + +// CHECK-LABEL: @vector_interleave_1d +// CHECK-SAME: %[[LHS:.*]]: vector<8xf32>, %[[RHS:.*]]: vector<8xf32>) +func.func @vector_interleave_1d(%a: vector<8xf32>, %b: vector<8xf32>) -> vector<16xf32> +{ + // CHECK: %[[ZIP:.*]] = llvm.shufflevector %[[LHS]], %[[RHS]] [0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32> + // CHECK: return %[[ZIP]] + %0 = vector.interleave %a, %b : vector<8xf32> + return %0 : vector<16xf32> +} + +// ----- + +// CHECK-LABEL: @vector_interleave_1d_scalable +// CHECK-SAME: %[[LHS:.*]]: vector<[4]xi32>, %[[RHS:.*]]: vector<[4]xi32>) +func.func @vector_interleave_1d_scalable(%a: vector<[4]xi32>, %b: vector<[4]xi32>) -> vector<[8]xi32> +{ + // CHECK: %[[ZIP:.*]] = "llvm.intr.experimental.vector.interleave2"(%[[LHS]], %[[RHS]]) : (vector<[4]xi32>, vector<[4]xi32>) -> vector<[8]xi32> + // CHECK: return %[[ZIP]] + %0 = vector.interleave %a, %b : vector<[4]xi32> + return %0 : vector<[8]xi32> +} -- GitLab From e678e6edec8d2672eb848b7453bd10e16cc54958 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Tue, 13 Feb 2024 10:49:19 +0000 Subject: [PATCH 046/284] [gn build] Port fe3406e34988 --- llvm/utils/gn/secondary/lld/ELF/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/lld/ELF/BUILD.gn b/llvm/utils/gn/secondary/lld/ELF/BUILD.gn index bd4a4f5d984b..d9037256e142 100644 --- a/llvm/utils/gn/secondary/lld/ELF/BUILD.gn +++ b/llvm/utils/gn/secondary/lld/ELF/BUILD.gn @@ -39,6 +39,7 @@ static_library("ELF") { "Arch/PPC64.cpp", "Arch/RISCV.cpp", "Arch/SPARCV9.cpp", + "Arch/SystemZ.cpp", "Arch/X86.cpp", "Arch/X86_64.cpp", "CallGraphSort.cpp", -- GitLab From 8456e0c290f759807023924481712767a19464c2 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 13 Feb 2024 11:03:20 +0000 Subject: [PATCH 047/284] [ADT] Allow std::next to work on BitVector's set_bits_iterator (#80830) Without this I would hit errors with libstdc++-12 like: /usr/include/c++/12/bits/stl_iterator_base_funcs.h:230:5: note: candidate template ignored: substitution failure [with _InputIterator = llvm::const_set_bits_iterator_impl]: argument may not have 'void' type next(_InputIterator __x, typename ^ --- llvm/include/llvm/ADT/BitVector.h | 2 +- llvm/unittests/ADT/BitVectorTest.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h index e0de1afcc941..0eaa77b6dd81 100644 --- a/llvm/include/llvm/ADT/BitVector.h +++ b/llvm/include/llvm/ADT/BitVector.h @@ -42,7 +42,7 @@ template class const_set_bits_iterator_impl { public: using iterator_category = std::forward_iterator_tag; - using difference_type = void; + using difference_type = std::ptrdiff_t; using value_type = int; using pointer = value_type*; using reference = value_type&; diff --git a/llvm/unittests/ADT/BitVectorTest.cpp b/llvm/unittests/ADT/BitVectorTest.cpp index e00e11e4655a..6a4780c143e5 100644 --- a/llvm/unittests/ADT/BitVectorTest.cpp +++ b/llvm/unittests/ADT/BitVectorTest.cpp @@ -1143,6 +1143,9 @@ TYPED_TEST(BitVectorTest, EmptyVectorGetData) { } TYPED_TEST(BitVectorTest, Iterators) { + TypeParam Singleton(1, true); + EXPECT_EQ(std::next(Singleton.set_bits_begin()), Singleton.set_bits_end()); + TypeParam Filled(10, true); EXPECT_NE(Filled.set_bits_begin(), Filled.set_bits_end()); unsigned Counter = 0; -- GitLab From 55d6643ccf6f9394d88d3d6359492000c58c2357 Mon Sep 17 00:00:00 2001 From: Pranav Bhandarkar <153014763+bhandarkar-pranav@users.noreply.github.com> Date: Tue, 13 Feb 2024 05:15:51 -0600 Subject: [PATCH 048/284] [mlir][openmp] - Add the depend clause to omp.target and related offloading directives (#81081) This patch adds support for the depend clause in a number of OpenMP directives/constructs related to offloading. Specifically, it adds the handling of the depend clause when it is used with the following constructs - target - target enter data - target update data - target exit data --- flang/lib/Lower/OpenMP.cpp | 5 +- mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 37 +++++++++-- mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp | 22 +++++-- mlir/test/Dialect/OpenMP/invalid.mlir | 37 +++++++++++ mlir/test/Dialect/OpenMP/ops.mlir | 63 ++++++++++++++++++- 5 files changed, 151 insertions(+), 13 deletions(-) diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp index e5887620d503..06850bebd7d0 100644 --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -2825,7 +2825,8 @@ genEnterExitUpdateDataOp(Fortran::lower::AbstractConverter &converter, directive); return firOpBuilder.create(currentLocation, ifClauseOperand, - deviceOperand, nowaitAttr, mapOperands); + deviceOperand, nullptr, mlir::ValueRange(), + nowaitAttr, mapOperands); } // This functions creates a block for the body of the targetOp's region. It adds @@ -3090,7 +3091,7 @@ genTargetOp(Fortran::lower::AbstractConverter &converter, auto targetOp = converter.getFirOpBuilder().create( currentLocation, ifClauseOperand, deviceOperand, threadLimitOperand, - nowaitAttr, mapOperands); + nullptr, mlir::ValueRange(), nowaitAttr, mapOperands); genBodyOfTargetOp(converter, semaCtx, eval, genNested, targetOp, mapSymTypes, mapSymLocs, mapSymbols, currentLocation); diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td index 44f3e5b8dbc3..c7a32de256e2 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td @@ -781,7 +781,7 @@ def ClauseTaskDependInOut : I32EnumAttrCase<"taskdependinout", 2>; def ClauseTaskDepend : I32EnumAttr< "ClauseTaskDepend", - "task depend clause", + "depend clause in a target or task construct", [ClauseTaskDependIn, ClauseTaskDependOut, ClauseTaskDependInOut]> { let genSpecializedAttr = 0; let cppNamespace = "::mlir::omp"; @@ -1447,11 +1447,17 @@ def Target_EnterDataOp: OpenMP_Op<"target_enter_data", The $map_types specifies the types and modifiers for the map clause. - TODO: depend clause and map_type_modifier values iterator and mapper. + The `depends` and `depend_vars` arguments are variadic lists of values + that specify the dependencies of this particular target task in relation to + other tasks. + + TODO: map_type_modifier values iterator and mapper. }]; let arguments = (ins Optional:$if_expr, Optional:$device, + OptionalAttr:$depends, + Variadic:$depend_vars, UnitAttr:$nowait, Variadic:$map_operands); @@ -1460,6 +1466,7 @@ def Target_EnterDataOp: OpenMP_Op<"target_enter_data", | `device` `(` $device `:` type($device) `)` | `nowait` $nowait | `map_entries` `(` $map_operands `:` type($map_operands) `)` + | `depend` `(` custom($depend_vars, type($depend_vars), $depends) `)` ) attr-dict }]; @@ -1494,11 +1501,17 @@ def Target_ExitDataOp: OpenMP_Op<"target_exit_data", The $map_types specifies the types and modifiers for the map clause. - TODO: depend clause and map_type_modifier values iterator and mapper. + The `depends` and `depend_vars` arguments are variadic lists of values + that specify the dependencies of this particular target task in relation to + other tasks. + + TODO: map_type_modifier values iterator and mapper. }]; let arguments = (ins Optional:$if_expr, Optional:$device, + OptionalAttr:$depends, + Variadic:$depend_vars, UnitAttr:$nowait, Variadic:$map_operands); @@ -1507,6 +1520,7 @@ def Target_ExitDataOp: OpenMP_Op<"target_exit_data", | `device` `(` $device `:` type($device) `)` | `nowait` $nowait | `map_entries` `(` $map_operands `:` type($map_operands) `)` + | `depend` `(` custom($depend_vars, type($depend_vars), $depends) `)` ) attr-dict }]; @@ -1545,11 +1559,16 @@ def Target_UpdateDataOp: OpenMP_Op<"target_update_data", during verification to make sure the restrictions for target update are respected. - TODO: depend clause + The `depends` and `depend_vars` arguments are variadic lists of values + that specify the dependencies of this particular target task in relation to + other tasks. + }]; let arguments = (ins Optional:$if_expr, Optional:$device, + OptionalAttr:$depends, + Variadic:$depend_vars, UnitAttr:$nowait, Variadic:$map_operands); @@ -1558,6 +1577,7 @@ def Target_UpdateDataOp: OpenMP_Op<"target_update_data", | `device` `(` $device `:` type($device) `)` | `nowait` $nowait | `motion_entries` `(` $map_operands `:` type($map_operands) `)` + | `depend` `(` custom($depend_vars, type($depend_vars), $depends) `)` ) attr-dict }]; @@ -1587,13 +1607,19 @@ def TargetOp : OpenMP_Op<"target",[IsolatedFromAbove, MapClauseOwningOpInterface The optional $nowait elliminates the implicit barrier so the parent task can make progress even if the target task is not yet completed. - TODO: is_device_ptr, depend, defaultmap, in_reduction + The `depends` and `depend_vars` arguments are variadic lists of values + that specify the dependencies of this particular target task in relation to + other tasks. + + TODO: is_device_ptr, defaultmap, in_reduction }]; let arguments = (ins Optional:$if_expr, Optional:$device, Optional:$thread_limit, + OptionalAttr:$depends, + Variadic:$depend_vars, UnitAttr:$nowait, Variadic:$map_operands); @@ -1605,6 +1631,7 @@ def TargetOp : OpenMP_Op<"target",[IsolatedFromAbove, MapClauseOwningOpInterface | `thread_limit` `(` $thread_limit `:` type($thread_limit) `)` | `nowait` $nowait | `map_entries` `(` custom($map_operands, type($map_operands)) `)` + | `depend` `(` custom($depend_vars, type($depend_vars), $depends) `)` ) $region attr-dict }]; diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index ef08bd87efc9..849449f9127d 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -628,7 +628,7 @@ static LogicalResult verifyDependVarList(Operation *op, return op->emitOpError() << "expected as many depend values" " as depend variables"; } else { - if (depends) + if (depends && !depends->empty()) return op->emitOpError() << "unexpected depend values"; return success(); } @@ -1032,19 +1032,31 @@ LogicalResult DataOp::verify() { } LogicalResult EnterDataOp::verify() { - return verifyMapClause(*this, getMapOperands()); + LogicalResult verifyDependVars = + verifyDependVarList(*this, getDepends(), getDependVars()); + return failed(verifyDependVars) ? verifyDependVars + : verifyMapClause(*this, getMapOperands()); } LogicalResult ExitDataOp::verify() { - return verifyMapClause(*this, getMapOperands()); + LogicalResult verifyDependVars = + verifyDependVarList(*this, getDepends(), getDependVars()); + return failed(verifyDependVars) ? verifyDependVars + : verifyMapClause(*this, getMapOperands()); } LogicalResult UpdateDataOp::verify() { - return verifyMapClause(*this, getMapOperands()); + LogicalResult verifyDependVars = + verifyDependVarList(*this, getDepends(), getDependVars()); + return failed(verifyDependVars) ? verifyDependVars + : verifyMapClause(*this, getMapOperands()); } LogicalResult TargetOp::verify() { - return verifyMapClause(*this, getMapOperands()); + LogicalResult verifyDependVars = + verifyDependVarList(*this, getDepends(), getDependVars()); + return failed(verifyDependVars) ? verifyDependVars + : verifyMapClause(*this, getMapOperands()); } //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir index 59b42390b206..1c1b6ea58e02 100644 --- a/mlir/test/Dialect/OpenMP/invalid.mlir +++ b/mlir/test/Dialect/OpenMP/invalid.mlir @@ -1651,6 +1651,15 @@ func.func @omp_target_enter_data(%map1: memref) { // ----- +func.func @omp_target_enter_data_depend(%a: memref) { + %0 = omp.map_info var_ptr(%a: memref, tensor) map_clauses(to) capture(ByRef) -> memref + // expected-error @below {{op expected as many depend values as depend variables}} + omp.target_enter_data map_entries(%0: memref ) {operandSegmentSizes = array} + return +} + +// ----- + func.func @omp_target_exit_data(%map1: memref) { %mapv = omp.map_info var_ptr(%map1 : memref, tensor) map_clauses(to) capture(ByRef) -> memref {name = ""} // expected-error @below {{from, release and delete map types are permitted}} @@ -1660,6 +1669,15 @@ func.func @omp_target_exit_data(%map1: memref) { // ----- +func.func @omp_target_exit_data_depend(%a: memref) { + %0 = omp.map_info var_ptr(%a: memref, tensor) map_clauses(from) capture(ByRef) -> memref + // expected-error @below {{op expected as many depend values as depend variables}} + omp.target_exit_data map_entries(%0: memref ) {operandSegmentSizes = array} + return +} + +// ----- + func.func @omp_target_update_invalid_motion_type(%map1 : memref) { %mapv = omp.map_info var_ptr(%map1 : memref, tensor) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref {name = ""} @@ -1732,6 +1750,25 @@ llvm.mlir.global internal @_QFsubEx() : i32 // ----- +func.func @omp_target_update_data_depend(%a: memref) { + %0 = omp.map_info var_ptr(%a: memref, tensor) map_clauses(to) capture(ByRef) -> memref + // expected-error @below {{op expected as many depend values as depend variables}} + omp.target_update_data motion_entries(%0: memref ) {operandSegmentSizes = array} + return +} + +// ----- + +func.func @omp_target_depend(%data_var: memref) { + // expected-error @below {{op expected as many depend values as depend variables}} + "omp.target"(%data_var) ({ + "omp.terminator"() : () -> () + }) {depends = [], operandSegmentSizes = array} : (memref) -> () + "func.return"() : () -> () +} + +// ----- + func.func @omp_distribute(%data_var : memref) -> () { // expected-error @below {{expected equal sizes for allocate and allocator variables}} "omp.distribute"(%data_var) <{operandSegmentSizes = array}> ({ diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir index 651405964c06..3bb4a288376e 100644 --- a/mlir/test/Dialect/OpenMP/ops.mlir +++ b/mlir/test/Dialect/OpenMP/ops.mlir @@ -517,7 +517,7 @@ func.func @omp_target(%if_cond : i1, %device : si32, %num_threads : i32, %map1: "omp.target"(%if_cond, %device, %num_threads) ({ // CHECK: omp.terminator omp.terminator - }) {nowait, operandSegmentSizes = array} : ( i1, si32, i32 ) -> () + }) {nowait, operandSegmentSizes = array} : ( i1, si32, i32 ) -> () // Test with optional map clause. // CHECK: %[[MAP_A:.*]] = omp.map_info var_ptr(%[[VAL_1:.*]] : memref, tensor) map_clauses(tofrom) capture(ByRef) -> memref {name = ""} @@ -1717,6 +1717,18 @@ func.func @omp_task_depend(%arg0: memref, %arg1: memref) { return } + +// CHECK-LABEL: @omp_target_depend +// CHECK-SAME: (%arg0: memref, %arg1: memref) { +func.func @omp_target_depend(%arg0: memref, %arg1: memref) { + // CHECK: omp.target depend(taskdependin -> %arg0 : memref, taskdependin -> %arg1 : memref, taskdependinout -> %arg0 : memref) { + omp.target depend(taskdependin -> %arg0 : memref, taskdependin -> %arg1 : memref, taskdependinout -> %arg0 : memref) { + // CHECK: omp.terminator + omp.terminator + } {operandSegmentSizes = array} + return +} + func.func @omp_threadprivate() { %0 = arith.constant 1 : i32 %1 = arith.constant 2 : i32 @@ -2145,3 +2157,52 @@ func.func @omp_targets_is_allocatable(%arg0: !llvm.ptr, %arg1: !llvm.ptr) -> () } return } + +// CHECK-LABEL: func @omp_target_enter_update_exit_data_depend +// CHECK-SAME:([[ARG0:%.*]]: memref, [[ARG1:%.*]]: memref, [[ARG2:%.*]]: memref) { +func.func @omp_target_enter_update_exit_data_depend(%a: memref, %b: memref, %c: memref) { +// CHECK-NEXT: [[MAP0:%.*]] = omp.map_info +// CHECK-NEXT: [[MAP1:%.*]] = omp.map_info +// CHECK-NEXT: [[MAP2:%.*]] = omp.map_info + %map_a = omp.map_info var_ptr(%a: memref, tensor) map_clauses(to) capture(ByRef) -> memref + %map_b = omp.map_info var_ptr(%b: memref, tensor) map_clauses(from) capture(ByRef) -> memref + %map_c = omp.map_info var_ptr(%c: memref, tensor) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref + + // Do some work on the host that writes to 'a' + omp.task depend(taskdependout -> %a : memref) { + "test.foo"(%a) : (memref) -> () + omp.terminator + } + + // Then map that over to the target + // CHECK: omp.target_enter_data nowait map_entries([[MAP0]], [[MAP2]] : memref, memref) depend(taskdependin -> [[ARG0]] : memref) + omp.target_enter_data nowait map_entries(%map_a, %map_c: memref, memref) depend(taskdependin -> %a: memref) + + // Compute 'b' on the target and copy it back + // CHECK: omp.target map_entries([[MAP1]] -> {{%.*}} : memref) { + omp.target map_entries(%map_b -> %arg0 : memref) { + ^bb0(%arg0: memref) : + "test.foo"(%arg0) : (memref) -> () + omp.terminator + } + + // Update 'a' on the host using 'b' + omp.task depend(taskdependout -> %a: memref){ + "test.bar"(%a, %b) : (memref, memref) -> () + } + + // Copy the updated 'a' onto the target + // CHECK: omp.target_update_data nowait motion_entries([[MAP0]] : memref) depend(taskdependin -> [[ARG0]] : memref) + omp.target_update_data motion_entries(%map_a : memref) depend(taskdependin -> %a : memref) nowait + + // Compute 'c' on the target and copy it back + %map_c_from = omp.map_info var_ptr(%c: memref, tensor) map_clauses(from) capture(ByRef) -> memref + omp.target map_entries(%map_a -> %arg0, %map_c_from -> %arg1 : memref, memref) depend(taskdependout -> %c : memref) { + ^bb0(%arg0 : memref, %arg1 : memref) : + "test.foobar"() : ()->() + omp.terminator + } + // CHECK: omp.target_exit_data map_entries([[MAP2]] : memref) depend(taskdependin -> [[ARG2]] : memref) + omp.target_exit_data map_entries(%map_c : memref) depend(taskdependin -> %c : memref) + return +} -- GitLab From e79ad7bb94611666a23764459098574bc0394d56 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Tue, 13 Feb 2024 11:09:06 +0000 Subject: [PATCH 049/284] [flang][docs] Fix a couple of warnings --- flang/docs/FortranLLVMTestSuite.md | 2 +- flang/docs/index.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flang/docs/FortranLLVMTestSuite.md b/flang/docs/FortranLLVMTestSuite.md index f07d415520a8..45485ef40106 100644 --- a/flang/docs/FortranLLVMTestSuite.md +++ b/flang/docs/FortranLLVMTestSuite.md @@ -62,7 +62,7 @@ cmake -G "Ninja" -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \ Tests from the gfortran test suite have been imported into the LLVM Test Suite. The tests will be run automatically if the test suite is built following the -instructions described [above](#running-the-LLVM-test-suite-with-fortran). +instructions described [above](#running-the-llvm-test-suite-with-fortran). There are additional configure-time options that can be used with the gfortran tests. More details about those options and their purpose can be found in [`Fortran/gfortran/README.md`](https://github.com/llvm/llvm-test-suite/tree/main/Fortran/gfortran/README.md)`. diff --git a/flang/docs/index.md b/flang/docs/index.md index ff8f4a22d5eb..d974a3628b01 100644 --- a/flang/docs/index.md +++ b/flang/docs/index.md @@ -68,6 +68,7 @@ on how to get in touch with us and to learn more about the current status. OpenACC OpenACC-descriptor-management.md OpenMP-4.5-grammar.md + OpenMP-descriptor-management OpenMP-semantics OptionComparison Overview -- GitLab From 97088b2ab2184ad4bd64f59fba0b92b70468b10d Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Tue, 13 Feb 2024 11:35:28 +0000 Subject: [PATCH 050/284] [RemoveDIs][ValueMapper] Remap DIAssignIDs in DPValues (#81595) Fix crash raised in comments for 5c9f7682b090124d9a8b69f92d3f7c269dca25fc --- llvm/lib/Transforms/Utils/ValueMapper.cpp | 1 + .../Generic/ipsccp-remap-assign-id.ll | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index a8ae3ee3b251..93a4c829df06 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -552,6 +552,7 @@ void Mapper::remapDPValue(DPValue &V) { V.setKillAddress(); else if (NewAddr) V.setAddress(NewAddr); + V.setAssignId(cast(mapMetadata(V.getAssignID()))); } // Find Value operands and remap those. diff --git a/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll b/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll new file mode 100644 index 000000000000..13ac88de5cde --- /dev/null +++ b/llvm/test/DebugInfo/Generic/ipsccp-remap-assign-id.ll @@ -0,0 +1,59 @@ +; RUN: opt -passes=ipsccp %s -S -o - | FileCheck %s +; RUN: opt --try-experimental-debuginfo-iterators -passes=ipsccp %s -S -o - | FileCheck %s + +;; Check the dbg.assign DIAssignID operand gets remapped after cloning. + +; CHECK: %tmp = alloca [4096 x i32], i32 0, align 16, !DIAssignID ![[ID1:[0-9]+]] +; CHECK-NEXT: dbg.assign(metadata i1 undef, metadata !{{.*}}, metadata !DIExpression(), metadata ![[ID1]], metadata ptr %tmp, metadata !DIExpression()) +; +; CHECK: %tmp = alloca [4096 x i32], i32 0, align 16, !DIAssignID ![[ID2:[0-9]+]] +; CHECK-NEXT: dbg.assign(metadata i1 undef, metadata !{{.*}}, metadata !DIExpression(), metadata ![[ID2]], metadata ptr %tmp, metadata !DIExpression()) + +; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) + +define void @inv_txfm_add_dct_dct_4x4_c() { +entry: + call void @inv_txfm_add_c(ptr @dav1d_inv_dct4_1d_c) + ret void +} + +declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) + +; Function Attrs: noinline +define void @inv_txfm_add_c(ptr %first_1d_fn) #2 { +entry: + %tmp = alloca [4096 x i32], i32 0, align 16, !DIAssignID !5 + tail call void @llvm.dbg.assign(metadata i1 undef, metadata !6, metadata !DIExpression(), metadata !5, metadata ptr %tmp, metadata !DIExpression()), !dbg !16 + call void @llvm.memset.p0.i64(ptr %tmp, i8 0, i64 0, i1 false), !DIAssignID !17 + call void %first_1d_fn(ptr null, i64 0, i32 0, i32 0) + ret void +} + +declare void @dav1d_inv_dct4_1d_c(ptr, i64, i32, i32) + +declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) + +attributes #2 = { noinline } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, splitDebugInlining: false, nameTableKind: GNU) +!1 = !DIFile(filename: "itx_tmpl.c", directory: ".") +!2 = !{} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{i32 7, !"debug-info-assignment-tracking", i1 true} +!5 = distinct !DIAssignID() +!6 = !DILocalVariable(name: "tmp", scope: !7, file: !1, line: 78, type: !10) +!7 = distinct !DISubprogram(name: "inv_txfm_add_c", scope: !1, file: !1, line: 41, type: !8, scopeLine: 45, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) +!8 = distinct !DISubroutineType(types: !9) +!9 = !{null} +!10 = !DICompositeType(tag: DW_TAG_array_type, baseType: !11, size: 131072, elements: !2) +!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "int32_t", file: !12, line: 26, baseType: !13) +!12 = !DIFile(filename: "stdint-intn.h", directory: ".") +!13 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int32_t", file: !14, line: 41, baseType: !15) +!14 = !DIFile(filename: "types.h", directory: ".") +!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!16 = !DILocation(line: 0, scope: !7) +!17 = distinct !DIAssignID() -- GitLab From bfc0b7c6891896ee8e9818f22800472510093864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Tue, 13 Feb 2024 12:10:54 +0000 Subject: [PATCH 051/284] [mlir][linalg] Document ops not supported by the vectoriser (nfc) (#81500) Adds a test to help document Linalg Ops that are currently not supported by the vectoriser (i.e. the logic to vectorise these is missing). The list is not exhaustive. --- .../Linalg/vectorization-unsupported.mlir | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 mlir/test/Dialect/Linalg/vectorization-unsupported.mlir diff --git a/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir b/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir new file mode 100644 index 000000000000..a1a52397386c --- /dev/null +++ b/mlir/test/Dialect/Linalg/vectorization-unsupported.mlir @@ -0,0 +1,73 @@ +// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics + +func.func @conv1d_nwc_wcf_dyn_ch_dim(%input: memref<4x6x?xf32>, %filter: memref<1x?x8xf32>, %output: memref<4x2x8xf32>) { + // expected-error @+1 {{Attempted to vectorize, but failed}} + linalg.conv_1d_nwc_wcf + {dilations = dense<1> : tensor<1xi64>, strides = dense<3> : tensor<1xi64>} + ins(%input, %filter : memref<4x6x?xf32>, memref<1x?x8xf32>) + outs(%output : memref<4x2x8xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.conv_1d_nwc_wcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 : !transform.any_op + transform.yield + } +} + +// ----- + +func.func @depthwise_conv1d_nwc_wc_dyn_ch_dim(%input: memref<3x5x?xf32>, %filter: memref<2x?xf32>, %output: memref<3x2x?xf32>) { + // expected-error @+1 {{Attempted to vectorize, but failed}} + linalg.depthwise_conv_1d_nwc_wc + {dilations = dense<2> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} + ins(%input, %filter : memref<3x5x?xf32>, memref<2x?xf32>) + outs(%output : memref<3x2x?xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 : !transform.any_op + transform.yield + } +} + +// ----- + +func.func @depthwise_conv1d_nwc_wc_dyn_w_dim(%input: memref<3x?x3xf32>, %filter: memref<2x3xf32>, %output: memref<3x?x3xf32>) { + // expected-error @+1 {{Attempted to vectorize, but failed}} + linalg.depthwise_conv_1d_nwc_wc + {dilations = dense<2> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} + ins(%input, %filter : memref<3x?x3xf32>, memref<2x3xf32>) + outs(%output : memref<3x?x3xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 : !transform.any_op + transform.yield + } +} + +// ----- + +func.func @conv1d_dyn_w_dim(%input: tensor, %filter: tensor<4xf32>, %output: tensor) -> tensor { + // expected-error @+1 {{Attempted to vectorize, but failed}} + %0 = linalg.conv_1d ins(%input, %filter : tensor, tensor<4xf32>) + outs(%output : tensor) -> tensor + return %0 : tensor +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.conv_1d"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 : !transform.any_op + transform.yield + } +} -- GitLab From 35ef3994bf738318b59ce640910fb1ccd3bb7dcb Mon Sep 17 00:00:00 2001 From: Ivan Butygin Date: Tue, 13 Feb 2024 15:30:58 +0300 Subject: [PATCH 052/284] [mlir][vector] ND vectors linearization pass (#81159) Common backends (LLVM, SPIR-V) only supports 1D vectors, LLVM conversion handles ND vectors (N >= 2) as `array>` and SPIR-V conversion doesn't handle them at all at the moment. Sometimes it's preferable to treat multidim vectors as linearized 1D. Add pass to do this. Only constants and simple elementwise ops are supported for now. @krzysz00 I've extracted yours result type conversion code from LegalizeToF32 and moved it to common place. Also, add ConversionPattern class operating on traits. --- .../Vector/Transforms/VectorRewritePatterns.h | 9 ++ .../mlir/Transforms/DialectConversion.h | 23 +++++ .../Dialect/Math/Transforms/LegalizeToF32.cpp | 20 ++-- .../Dialect/Vector/Transforms/CMakeLists.txt | 1 + .../Vector/Transforms/VectorLinearize.cpp | 97 +++++++++++++++++++ .../Transforms/Utils/DialectConversion.cpp | 21 ++++ mlir/test/Dialect/Vector/linearize.mlir | 19 ++++ .../Dialect/Vector/TestVectorTransforms.cpp | 29 ++++++ 8 files changed, 206 insertions(+), 13 deletions(-) create mode 100644 mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp create mode 100644 mlir/test/Dialect/Vector/linearize.mlir diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h index f5941d32e683..7c943f07066c 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h @@ -20,7 +20,9 @@ #include "mlir/Dialect/Vector/Transforms/VectorTransformsEnums.h.inc" namespace mlir { +class ConversionTarget; class RewritePatternSet; +class TypeConverter; namespace arith { class AndIOp; @@ -375,6 +377,13 @@ void populateVectorNarrowTypeRewritePatterns(RewritePatternSet &patterns, void populateVectorTransposeNarrowTypeRewritePatterns( RewritePatternSet &patterns, PatternBenefit benefit = 1); +/// Populates patterns for ND vectors (N >= 2) linearization and sets up the +/// provided ConversionTarget with the appropriate legality configuration for +/// the ops to get converted properly. +void populateVectorLinearizeTypeConversionsAndLegality( + TypeConverter &typeConverter, RewritePatternSet &patterns, + ConversionTarget &target); + } // namespace vector } // namespace mlir diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h index b1ec1fe4ecd5..091131651bbf 100644 --- a/mlir/include/mlir/Transforms/DialectConversion.h +++ b/mlir/include/mlir/Transforms/DialectConversion.h @@ -604,6 +604,29 @@ private: using ConversionPattern::matchAndRewrite; }; +/// OpTraitConversionPattern is a wrapper around ConversionPattern that allows +/// for matching and rewriting against instances of an operation that possess a +/// given trait. +template